00000001:01000000:0.0F:1787827083.569355:0:2333:0:(lnet-crypto.c:359:cfs_crypto_performance_test()) Crypto hash algorithm adler32 speed = 1479 MB/s 00000001:01000000:0.0:1787827083.819508:0:2333:0:(lnet-crypto.c:359:cfs_crypto_performance_test()) Crypto hash algorithm crc32 speed = 1807 MB/s 00000001:01000000:0.0:1787827084.069346:0:2333:0:(lnet-crypto.c:359:cfs_crypto_performance_test()) Crypto hash algorithm crc32c speed = 11183 MB/s 00000020:02000000:3.0F:1787827084.298672:0:2362:0:(class_obd.c:771:obdclass_init()) Lustre: Build Version: 2.17.57_65_g89f8e4a 00000400:02000000:3.0:1787827084.608978:0:2377:0:(api-ni.c:2823:lnet_startup_lndni()) Added LNI 192.168.203.29@tcp [8/256/0/180] 00008000:02000000:0.0:1787827086.844512:0:2527:0:(echo_client.c:2574:obdecho_init()) Echo OBD driver; http://www.lustre.org/ 00000020:01000004:2.0F:1787827148.982134:0:6141:0:(obd_mount.c:923:lmd_print()) mount data: 00000020:01000004:2.0:1787827148.982143:0:6141:0:(obd_mount.c:925:lmd_print()) profile: lustre-client 00000020:01000004:2.0:1787827148.982144:0:6141:0:(obd_mount.c:926:lmd_print()) device: 192.168.203.129@tcp:/lustre 00000020:01000004:2.0:1787827148.982146:0:6141:0:(obd_mount.c:929:lmd_print()) options: user_xattr,flock 00000080:01200004:2.0:1787827148.982255:0:6141:0:(super25.c:131:lustre_fill_super()) VFS Op: sb 00000000d0fa1854 00000080:01000004:2.0:1787827148.982299:0:6141:0:(super25.c:162:lustre_fill_super()) Mounting client lustre-client 00000020:01000004:2.0:1787827148.982313:0:6141:0:(obd_mount.c:411:lustre_start_mgc()) Start MGC 'MGC192.168.203.129@tcp' 00000020:01000004:2.0:1787827148.982358:0:6141:0:(obd_mount.c:230:lustre_start_simple()) Starting OBD MGC192.168.203.129@tcp (typ=mgc) 00000020:00000080:2.0:1787827148.982379:0:6141:0:(genops.c:413:class_newdev()) Allocate new device MGC192.168.203.129@tcp (00000000fcf2d05a) 00000020:00000080:2.0:1787827148.982393:0:6141:0:(obd_config.c:720:class_attach_name()) OBD: dev 0 attached type mgc with refcount 2 00000100:00100000:2.0:1787827148.982623:0:6141:0:(service.c:155:ptlrpc_grow_req_bufs()) ldlm_cbd: allocate 1 new 8192-byte reqbufs (0/1 left), rc = 0 00000100:00100000:2.0:1787827148.982643:0:6141:0:(service.c:155:ptlrpc_grow_req_bufs()) ldlm_cbd: allocate 1 new 8192-byte reqbufs (0/1 left), rc = 0 00000100:00100000:2.0:1787827148.982769:0:6141:0:(service.c:3462:ptlrpc_start_thread()) ldlm_cbd[0] started 0 min 2 max 34 00000100:00100000:2.0:1787827148.982774:0:6141:0:(service.c:3521:ptlrpc_start_thread()) starting thread 'ldlm_cb00_000' 00000100:00100000:2.0:1787827148.983120:0:6141:0:(service.c:3462:ptlrpc_start_thread()) ldlm_cbd[0] started 1 min 2 max 34 00000100:00100000:2.0:1787827148.983127:0:6141:0:(service.c:3521:ptlrpc_start_thread()) starting thread 'ldlm_cb00_001' 00000100:00100000:2.0:1787827148.983495:0:6141:0:(service.c:3462:ptlrpc_start_thread()) ldlm_cbd[1] started 0 min 2 max 34 00000100:00100000:2.0:1787827148.983502:0:6141:0:(service.c:3521:ptlrpc_start_thread()) starting thread 'ldlm_cb01_000' 00000100:00100000:0.0:1787827148.983709:0:6141:0:(service.c:3462:ptlrpc_start_thread()) ldlm_cbd[1] started 1 min 2 max 34 00000100:00100000:0.0:1787827148.983716:0:6141:0:(service.c:3521:ptlrpc_start_thread()) starting thread 'ldlm_cb01_001' 00000100:00100000:0.0:1787827148.984172:0:6141:0:(service.c:155:ptlrpc_grow_req_bufs()) ldlm_canceld: allocate 64 new 8192-byte reqbufs (0/64 left), rc = 0 00000100:00100000:0.0:1787827148.984540:0:6141:0:(service.c:155:ptlrpc_grow_req_bufs()) ldlm_canceld: allocate 64 new 8192-byte reqbufs (0/64 left), rc = 0 00000100:00100000:0.0:1787827148.984694:0:6141:0:(service.c:3462:ptlrpc_start_thread()) ldlm_canceld[0] started 0 min 3 max 34 00000100:00100000:0.0:1787827148.984700:0:6141:0:(service.c:3521:ptlrpc_start_thread()) starting thread 'ldlm_cn00_000' 00000100:00100000:2.0:1787827148.985394:0:6141:0:(service.c:3462:ptlrpc_start_thread()) ldlm_canceld[0] started 1 min 3 max 34 00000100:00100000:2.0:1787827148.985401:0:6141:0:(service.c:3521:ptlrpc_start_thread()) starting thread 'ldlm_cn00_001' 00000100:00100000:2.0:1787827148.985899:0:6141:0:(service.c:3462:ptlrpc_start_thread()) ldlm_canceld[0] started 2 min 3 max 34 00000100:00100000:2.0:1787827148.985907:0:6141:0:(service.c:3521:ptlrpc_start_thread()) starting thread 'ldlm_cn00_002' 00000100:00100000:3.0:1787827148.986303:0:6141:0:(service.c:3462:ptlrpc_start_thread()) ldlm_canceld[1] started 0 min 3 max 34 00000100:00100000:0.0:1787827148.986347:0:6141:0:(service.c:3521:ptlrpc_start_thread()) starting thread 'ldlm_cn01_000' 00000100:00100000:2.0:1787827148.986841:0:6141:0:(service.c:3462:ptlrpc_start_thread()) ldlm_canceld[1] started 1 min 3 max 34 00000100:00100000:2.0:1787827148.986851:0:6141:0:(service.c:3521:ptlrpc_start_thread()) starting thread 'ldlm_cn01_001' 00000100:00100000:0.0:1787827148.987494:0:6141:0:(service.c:3462:ptlrpc_start_thread()) ldlm_canceld[1] started 2 min 3 max 34 00000100:00100000:0.0:1787827148.987504:0:6141:0:(service.c:3521:ptlrpc_start_thread()) starting thread 'ldlm_cn01_002' 00010000:00010000:2.0:1787827148.987988:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787827148.987993:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787827148.987996:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787827148.988164:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787827148.988168:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787827148.988170:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00080000:3.0:1787827148.988541:0:6141:0:(ldlm_lib.c:102:import_set_conn()) imp 00000000a47aacb7@MGC192.168.203.129@tcp: add connection 192.168.203.129@tcp at head 00010000:00010000:3.0:1787827148.988689:0:6141:0:(ldlm_pool.c:923:ldlm_pool_init()) Lock pool ldlm-pool-MGC192.168.203.129@tcp-0 is initialized 00000040:01000000:3.0:1787827148.988695:0:6141:0:(llog_obd.c:197:llog_setup()) obd MGC192.168.203.129@tcp ctxt 1 is initialized 10000000:01000000:0.0:1787827148.989322:0:6155:0:(mgc_request.c:631:mgc_requeue_thread()) Starting requeue thread 00000020:00000080:3.0:1787827148.989371:0:6141:0:(obd_config.c:826:class_setup()) finished setup of obd MGC192.168.203.129@tcp (uuid 0bb8b105-f59c-455a-b6b1-ce40efc3999c) 00000020:00000080:3.0:1787827148.989387:0:6141:0:(genops.c:1385:class_connect()) connect: client 0bb8b105-f59c-455a-b6b1-ce40efc3999c, cookie 0x218c3ba1346960d 00000100:00080000:3.0:1787827148.989397:0:6141:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 0 00000100:00080000:3.0:1787827148.989402:0:6141:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import 00000000a47aacb7 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:3.0:1787827148.989426:0:6141:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000f7d4dedf x1874672572432512/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787827148.989442:0:6141:0:(pinger.c:373:ptlrpc_pinger_add_import()) adding pingable import 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS 00000080:00200000:3.0:1787827148.989458:0:6141:0:(llite_lib.c:1506:ll_fill_super()) VFS Op: cfg_instance lustre-client-ffff8cb98543f000 (sb 00000000d0fa1854) 00000100:00100000:0.0:1787827148.989526:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@00000000f7d4dedf pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572432512:192.168.203.129@tcp:250: 00000080:01000000:3.0:1787827148.989558:0:6141:0:(llite_lib.c:1232:ll_options()) Parsing opts user_xattr,flock 00000080:00000004:3.0:1787827148.989562:0:6141:0:(llite_lib.c:1242:ll_options()) next opt=user_xattr 00000080:00000004:3.0:1787827148.989566:0:6141:0:(llite_lib.c:1242:ll_options()) next opt=flock 00000080:01000000:3.0:1787827148.989571:0:6141:0:(llite_lib.c:1540:ll_fill_super()) llite sb uuid: 8cef8de3-2326-42de-9d41-f4cb057ed6e8 10000000:01000000:3.0:1787827148.990288:0:6141:0:(mgc_request.c:1944:mgc_process_config()) parse_log lustre-client from 0 10000000:01000000:3.0:1787827148.990292:0:6141:0:(mgc_request.c:329:config_log_add()) add config log lustre-client-ffff8cb98543f000 10000000:01000000:3.0:1787827148.990296:0:6141:0:(mgc_request.c:194:do_config_log_add()) do adding config log lustre-sptlrpc-ffff8cb9881c0000 10000000:01000000:3.0:1787827148.990300:0:6141:0:(mgc_request.c:68:mgc_name2resid()) log lustre-sptlrpc to resid 0x65727473756c/0x0 (lustre) 10000000:01000000:3.0:1787827148.990308:0:6141:0:(mgc_request.c:1802:mgc_process_log()) Process log lustre-sptlrpc-ffff8cb9881c0000 from 1 10000000:01000000:3.0:1787827148.990311:0:6141:0:(mgc_request.c:996:mgc_enqueue()) Enqueue for lustre-sptlrpc (res 0x65727473756c) 00010000:00010000:3.0:1787827148.990340:0:6141:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: MGC192.168.203.129@tcp lock: 000000003e7047f3/0x218c3ba13469614 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 6141 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787827148.990348:0:6141:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000000000000 ns: MGC192.168.203.129@tcp lock: 000000003e7047f3/0x218c3ba13469614 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 6141 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787827148.990360:0:6141:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: MGC192.168.203.129@tcp lock: 000000003e7047f3/0x218c3ba13469614 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 6141 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00080000:3.0:1787827148.990372:0:6141:0:(client.c:1845:ptlrpc_send_new_req()) @@@ req waiting for recovery: (FULL != CONNECTING) req@000000002fce079b x1874672572432640/t0(0) o101->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl Rpc:WQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827148.990380:0:6141:0:(client.c:2715:ptlrpc_set_wait()) set 00000000f7cb5ab9 going to sleep for 0 seconds 00000100:00100000:2.0:1787827148.990698:0:2393:0:(client.c:3125:ptlrpc_free_committed()) MGC192.168.203.129@tcp: committing for last_committed 0 gen 1 00000100:00080000:2.0:1787827148.990705:0:2393:0:(import.c:1132:ptlrpc_connect_interpret()) MGC192.168.203.129@tcp: connect to target with instance 0 00000100:00080000:2.0:1787827148.990710:0:2393:0:(import.c:956:ptlrpc_connect_set_flags()) MGC192.168.203.129@tcp: Resetting ns_connect_flags to server flags: 0xa000011001002060 10000000:01000000:2.0:1787827148.990713:0:2393:0:(mgc_request.c:1168:mgc_import_event()) import event 0x808005 00000100:00080000:2.0:1787827148.990716:0:2393:0:(import.c:65:import_set_state_nolock()) 00000000a47aacb7 MGS: changing import state from CONNECTING to FULL 10000000:01000000:2.0:1787827148.990718:0:2393:0:(mgc_request.c:1168:mgc_import_event()) import event 0x808004 00000100:00080000:2.0:1787827148.990726:0:2393:0:(pinger.c:186:ptlrpc_pinger_ir_up()) IR up 00000100:00080000:2.0:1787827148.990731:0:2393:0:(recover.c:191:ptlrpc_wake_delayed()) @@@ waking (set 00000000f7cb5ab9): req@000000002fce079b x1874672572432640/t0(0) o101->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl Rpc:WQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787827148.990747:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@00000000f7d4dedf pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572432512:192.168.203.129@tcp:250: 00000100:00100000:3.0:1787827148.990803:0:6141:0:(client.c:2715:ptlrpc_set_wait()) set 00000000f7cb5ab9 going to sleep for 11 seconds 00010000:00010000:2.0:1787827148.991364:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827148.991464:0:6141:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@000000002fce079b pname:cluuid:pid:xid:nid:opc:job mount.lustre:0bb8b105-f59c-455a-b6b1-ce40efc3999c:6141:1874672572432640:192.168.203.129@tcp:101: 00010000:00010000:3.0:1787827148.991493:0:6141:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: MGC192.168.203.129@tcp lock: 000000003e7047f3/0x218c3ba13469614 lrc: 4/1,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x11000000000000 nid: local remote: 0xb2e8bd65d71dd9f0 expref: -99 pid: 6141 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787827148.991503:0:6141:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: MGC192.168.203.129@tcp lock: 000000003e7047f3/0x218c3ba13469614 lrc: 4/1,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0xb2e8bd65d71dd9f0 expref: -99 pid: 6141 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1787827148.991566:0:6141:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@000000002fce079b pname:cluuid:pid:xid:nid:opc:job mount.lustre:0bb8b105-f59c-455a-b6b1-ce40efc3999c:6141:1874672572432768:192.168.203.129@tcp:501: 00000100:00100000:3.0:1787827148.991612:0:6141:0:(client.c:2715:ptlrpc_set_wait()) set 00000000f7cb5ab9 going to sleep for 11 seconds 00000100:00100000:3.0:1787827148.992345:0:6141:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@000000002fce079b pname:cluuid:pid:xid:nid:opc:job mount.lustre:0bb8b105-f59c-455a-b6b1-ce40efc3999c:6141:1874672572432768:192.168.203.129@tcp:501: 10000000:01000000:3.0:1787827148.992372:0:6141:0:(mgc_request.c:1879:mgc_process_log()) MGC192.168.203.129@tcp: configuration from log 'lustre-sptlrpc' failed (-2). 00010000:00010000:3.0:1787827148.992388:0:6141:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: MGC192.168.203.129@tcp lock: 000000003e7047f3/0x218c3ba13469614 lrc: 3/1,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x11000000000000 nid: local remote: 0xb2e8bd65d71dd9f0 expref: -99 pid: 6141 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787827148.992396:0:6141:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: MGC192.168.203.129@tcp lock: 000000003e7047f3/0x218c3ba13469614 lrc: 2/0,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x11000000000000 nid: local remote: 0xb2e8bd65d71dd9f0 expref: -99 pid: 6141 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:01000000:3.0:1787827148.992403:0:6141:0:(mgc_request.c:194:do_config_log_add()) do adding config log params-ffff8cb98543f000 10000000:01000000:3.0:1787827148.992408:0:6141:0:(mgc_request.c:68:mgc_name2resid()) log params to resid 0x736d61726170/0x3 (params) 10000000:01000000:3.0:1787827148.992419:0:6141:0:(mgc_request.c:194:do_config_log_add()) do adding config log lustre-client-ffff8cb98543f000 10000000:01000000:3.0:1787827148.992422:0:6141:0:(mgc_request.c:68:mgc_name2resid()) log lustre-client to resid 0x65727473756c/0x0 (lustre) 10000000:01000000:3.0:1787827148.992426:0:6141:0:(mgc_request.c:194:do_config_log_add()) do adding config log lustre-cliir-ffff8cb98543f000 10000000:01000000:3.0:1787827148.992428:0:6141:0:(mgc_request.c:68:mgc_name2resid()) log lustre-cliir to resid 0x65727473756c/0x2 (lustre) 10000000:01000000:3.0:1787827148.992432:0:6141:0:(mgc_request.c:1802:mgc_process_log()) Process log lustre-client-ffff8cb98543f000 from 1 10000000:01000000:3.0:1787827148.992434:0:6141:0:(mgc_request.c:996:mgc_enqueue()) Enqueue for lustre-client (res 0x65727473756c) 00010000:00010000:3.0:1787827148.992453:0:6141:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: MGC192.168.203.129@tcp lock: 000000005a726d24/0x218c3ba1346961b lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 6141 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787827148.992460:0:6141:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000000000000 ns: MGC192.168.203.129@tcp lock: 000000005a726d24/0x218c3ba1346961b lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 6141 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787827148.992467:0:6141:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: MGC192.168.203.129@tcp lock: 000000005a726d24/0x218c3ba1346961b lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 6141 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1787827148.992493:0:6141:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@000000002fce079b pname:cluuid:pid:xid:nid:opc:job mount.lustre:0bb8b105-f59c-455a-b6b1-ce40efc3999c:6141:1874672572432896:192.168.203.129@tcp:101: 00000100:00100000:3.0:1787827148.992535:0:6141:0:(client.c:2715:ptlrpc_set_wait()) set 00000000f7cb5ab9 going to sleep for 11 seconds 00000100:00100000:3.0:1787827148.993746:0:6141:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@000000002fce079b pname:cluuid:pid:xid:nid:opc:job mount.lustre:0bb8b105-f59c-455a-b6b1-ce40efc3999c:6141:1874672572432896:192.168.203.129@tcp:101: 00010000:00010000:3.0:1787827148.993765:0:6141:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: MGC192.168.203.129@tcp lock: 000000005a726d24/0x218c3ba1346961b lrc: 4/1,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x11000000000000 nid: local remote: 0xb2e8bd65d71dd9f7 expref: -99 pid: 6141 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787827148.993773:0:6141:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: MGC192.168.203.129@tcp lock: 000000005a726d24/0x218c3ba1346961b lrc: 4/1,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x1000000000000 nid: local remote: 0xb2e8bd65d71dd9f7 expref: -99 pid: 6141 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1787827148.993830:0:6141:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@000000002fce079b pname:cluuid:pid:xid:nid:opc:job mount.lustre:0bb8b105-f59c-455a-b6b1-ce40efc3999c:6141:1874672572433024:192.168.203.129@tcp:501: 00000100:00100000:3.0:1787827148.993949:0:6141:0:(client.c:2715:ptlrpc_set_wait()) set 00000000f7cb5ab9 going to sleep for 11 seconds 00000100:00100000:3.0:1787827148.995044:0:6141:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@000000002fce079b pname:cluuid:pid:xid:nid:opc:job mount.lustre:0bb8b105-f59c-455a-b6b1-ce40efc3999c:6141:1874672572433024:192.168.203.129@tcp:501: 00000100:00100000:3.0:1787827148.995081:0:6141:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@000000002fce079b pname:cluuid:pid:xid:nid:opc:job mount.lustre:0bb8b105-f59c-455a-b6b1-ce40efc3999c:6141:1874672572433152:192.168.203.129@tcp:503: 00000100:00100000:3.0:1787827148.995150:0:6141:0:(client.c:2715:ptlrpc_set_wait()) set 00000000f7cb5ab9 going to sleep for 11 seconds 00000100:00100000:3.0:1787827148.997023:0:6141:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@000000002fce079b pname:cluuid:pid:xid:nid:opc:job mount.lustre:0bb8b105-f59c-455a-b6b1-ce40efc3999c:6141:1874672572433152:192.168.203.129@tcp:503: 00000100:00100000:3.0:1787827148.997426:0:6160:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@000000002fce079b pname:cluuid:pid:xid:nid:opc:job llog_process_th:0bb8b105-f59c-455a-b6b1-ce40efc3999c:6160:1874672572433280:192.168.203.129@tcp:502: 00000100:00100000:3.0:1787827148.997509:0:6160:0:(client.c:2715:ptlrpc_set_wait()) set 000000004a11bf1c going to sleep for 11 seconds 00000100:00100000:3.0:1787827148.999767:0:6160:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@000000002fce079b pname:cluuid:pid:xid:nid:opc:job llog_process_th:0bb8b105-f59c-455a-b6b1-ce40efc3999c:6160:1874672572433280:192.168.203.129@tcp:502: 00000020:01000000:3.0:1787827148.999801:0:6160:0:(obd_config.c:1893:class_config_llog_handler()) Marker, inst_flg=0x0 mark_flg=0x1 00000020:00000080:3.0:1787827148.999811:0:6160:0:(obd_config.c:1472:class_process_config()) processing cmd: cf010 00000020:00000080:3.0:1787827148.999814:0:6160:0:(obd_config.c:1563:class_process_config()) marker 4 (0x1) lustre-clilov lov setup 00000020:01000000:3.0:1787827148.999821:0:6160:0:(obd_config.c:2004:class_config_llog_handler()) cmd cf001, instance name: lustre-clilov-ffff8cb98543f000 00000020:00000080:3.0:1787827148.999825:0:6160:0:(obd_config.c:1472:class_process_config()) processing cmd: cf001 00000020:00000080:3.0:1787827148.999844:0:6160:0:(genops.c:413:class_newdev()) Allocate new device lustre-clilov-ffff8cb98543f000 (00000000e995b3b8) 00000020:00000080:3.0:1787827148.999858:0:6160:0:(obd_config.c:720:class_attach_name()) OBD: dev 1 attached type lov with refcount 2 00000020:01000000:3.0:1787827148.999864:0:6160:0:(obd_config.c:2004:class_config_llog_handler()) cmd cf003, instance name: lustre-clilov-ffff8cb98543f000 00000020:00000080:3.0:1787827148.999866:0:6160:0:(obd_config.c:1472:class_process_config()) processing cmd: cf003 00000020:00000080:3.0:1787827149.000232:0:6160:0:(obd_config.c:826:class_setup()) finished setup of obd lustre-clilov-ffff8cb98543f000 (uuid 8cef8de3-2326-42de-9d41-f4cb057ed6e8) 00000020:01000000:3.0:1787827149.000237:0:6160:0:(obd_config.c:1893:class_config_llog_handler()) Marker, inst_flg=0x2 mark_flg=0x2 00000020:00000080:3.0:1787827149.000241:0:6160:0:(obd_config.c:1472:class_process_config()) processing cmd: cf010 00000020:00000080:3.0:1787827149.000243:0:6160:0:(obd_config.c:1563:class_process_config()) marker 4 (0x2) lustre-clilov lov setup 00000020:01000000:3.0:1787827149.000246:0:6160:0:(obd_config.c:1893:class_config_llog_handler()) Marker, inst_flg=0x0 mark_flg=0x1 00000020:00000080:3.0:1787827149.000250:0:6160:0:(obd_config.c:1472:class_process_config()) processing cmd: cf010 00000020:00000080:3.0:1787827149.000251:0:6160:0:(obd_config.c:1563:class_process_config()) marker 5 (0x1) lustre-clilmv lmv setup 00000020:01000000:3.0:1787827149.000255:0:6160:0:(obd_config.c:2004:class_config_llog_handler()) cmd cf001, instance name: lustre-clilmv-ffff8cb98543f000 00000020:00000080:3.0:1787827149.000257:0:6160:0:(obd_config.c:1472:class_process_config()) processing cmd: cf001 00000020:00000080:3.0:1787827149.000283:0:6160:0:(genops.c:413:class_newdev()) Allocate new device lustre-clilmv-ffff8cb98543f000 (00000000a6b6e777) 00000020:00000080:3.0:1787827149.000303:0:6160:0:(obd_config.c:720:class_attach_name()) OBD: dev 2 attached type lmv with refcount 2 00000020:01000000:3.0:1787827149.000310:0:6160:0:(obd_config.c:2004:class_config_llog_handler()) cmd cf003, instance name: lustre-clilmv-ffff8cb98543f000 00000020:00000080:3.0:1787827149.000313:0:6160:0:(obd_config.c:1472:class_process_config()) processing cmd: cf003 00000020:00000080:3.0:1787827149.000540:0:6160:0:(obd_config.c:826:class_setup()) finished setup of obd lustre-clilmv-ffff8cb98543f000 (uuid 8cef8de3-2326-42de-9d41-f4cb057ed6e8) 00000020:01000000:3.0:1787827149.000551:0:6160:0:(obd_config.c:1893:class_config_llog_handler()) Marker, inst_flg=0x2 mark_flg=0x2 00000020:00000080:3.0:1787827149.000555:0:6160:0:(obd_config.c:1472:class_process_config()) processing cmd: cf010 00000020:00000080:3.0:1787827149.000557:0:6160:0:(obd_config.c:1563:class_process_config()) marker 5 (0x2) lustre-clilmv lmv setup 00000020:01000000:3.0:1787827149.000560:0:6160:0:(obd_config.c:1893:class_config_llog_handler()) Marker, inst_flg=0x0 mark_flg=0x1 00000020:00000080:3.0:1787827149.000563:0:6160:0:(obd_config.c:1472:class_process_config()) processing cmd: cf010 00000020:00000080:3.0:1787827149.000565:0:6160:0:(obd_config.c:1563:class_process_config()) marker 6 (0x1) lustre-client mount opts 00000020:00000080:3.0:1787827149.000569:0:6160:0:(obd_config.c:1472:class_process_config()) processing cmd: cf007 00000020:00000080:3.0:1787827149.000571:0:6160:0:(obd_config.c:1516:class_process_config()) mountopt: profile lustre-client osc lustre-clilov mdc lustre-clilmv 00000020:01000000:3.0:1787827149.000574:0:6160:0:(obd_config.c:1149:class_add_profile()) Add profile lustre-client 00000020:01000000:3.0:1787827149.000581:0:6160:0:(obd_config.c:1893:class_config_llog_handler()) Marker, inst_flg=0x2 mark_flg=0x2 00000020:00000080:3.0:1787827149.000583:0:6160:0:(obd_config.c:1472:class_process_config()) processing cmd: cf010 00000020:00000080:3.0:1787827149.000585:0:6160:0:(obd_config.c:1563:class_process_config()) marker 6 (0x2) lustre-client mount opts 00000020:01000000:3.0:1787827149.000589:0:6160:0:(obd_config.c:1893:class_config_llog_handler()) Marker, inst_flg=0x0 mark_flg=0x1 00000020:00000080:3.0:1787827149.000593:0:6160:0:(obd_config.c:1472:class_process_config()) processing cmd: cf010 00000020:00000080:3.0:1787827149.000595:0:6160:0:(obd_config.c:1563:class_process_config()) marker 7 (0x1) lustre-MDT0000 add mdc 00000020:00000080:3.0:1787827149.000599:0:6160:0:(obd_config.c:1472:class_process_config()) processing cmd: cf005 00000020:00000080:3.0:1787827149.000604:0:6160:0:(obd_config.c:1487:class_process_config()) adding mapping from uuid 192.168.203.129@tcp to nid 0x20000c0a8cb81 (192.168.203.129@tcp) 00000020:01000000:3.0:1787827149.000631:0:6160:0:(obd_config.c:2004:class_config_llog_handler()) cmd cf001, instance name: lustre-MDT0000-mdc-ffff8cb98543f000 00000020:00000080:3.0:1787827149.000635:0:6160:0:(obd_config.c:1472:class_process_config()) processing cmd: cf001 00000020:00000080:3.0:1787827149.000642:0:6160:0:(genops.c:413:class_newdev()) Allocate new device lustre-MDT0000-mdc-ffff8cb98543f000 (00000000c2c35c3c) 00000020:00000080:3.0:1787827149.000652:0:6160:0:(obd_config.c:720:class_attach_name()) OBD: dev 3 attached type mdc with refcount 2 00000020:01000000:3.0:1787827149.000658:0:6160:0:(obd_config.c:2004:class_config_llog_handler()) cmd cf003, instance name: lustre-MDT0000-mdc-ffff8cb98543f000 00000020:00000080:3.0:1787827149.000660:0:6160:0:(obd_config.c:1472:class_process_config()) processing cmd: cf003 00010000:00080000:3.0:1787827149.000719:0:6160:0:(ldlm_lib.c:102:import_set_conn()) imp 000000004c3d71cf@lustre-MDT0000-mdc-ffff8cb98543f000: add connection 192.168.203.129@tcp at head 00010000:00010000:2.0:1787827149.002550:0:6160:0:(ldlm_pool.c:923:ldlm_pool_init()) Lock pool ldlm-pool-lustre-MDT0000-mdc-ffff8cb98543f000-1 is initialized 00000040:01000000:2.0:1787827149.002817:0:6160:0:(llog_obd.c:197:llog_setup()) obd lustre-MDT0000-mdc-ffff8cb98543f000 ctxt 13 is initialized 00000020:00000080:2.0:1787827149.003056:0:6160:0:(obd_config.c:826:class_setup()) finished setup of obd lustre-MDT0000-mdc-ffff8cb98543f000 (uuid 8cef8de3-2326-42de-9d41-f4cb057ed6e8) 00000020:01000000:2.0:1787827149.003066:0:6160:0:(obd_config.c:2004:class_config_llog_handler()) cmd cf014, instance name: lustre-clilmv-ffff8cb98543f000 00000020:00000080:2.0:1787827149.003071:0:6160:0:(obd_config.c:1472:class_process_config()) processing cmd: cf014 00800000:01000000:2.0:1787827149.003102:0:6160:0:(lmv_obd.c:442:lmv_add_target()) Target uuid: lustre-MDT0000_UUID. index 0 00000020:01000000:2.0:1787827149.003152:0:6160:0:(obd_config.c:1893:class_config_llog_handler()) Marker, inst_flg=0x2 mark_flg=0x2 00000020:00000080:2.0:1787827149.003157:0:6160:0:(obd_config.c:1472:class_process_config()) processing cmd: cf010 00000020:00000080:2.0:1787827149.003159:0:6160:0:(obd_config.c:1563:class_process_config()) marker 7 (0x2) lustre-MDT0000 add mdc 00000020:01000000:2.0:1787827149.003162:0:6160:0:(obd_config.c:1893:class_config_llog_handler()) Marker, inst_flg=0x0 mark_flg=0x5 00000020:01000000:2.0:1787827149.003165:0:6160:0:(obd_config.c:1902:class_config_llog_handler()) SKIP #9 00000020:01000000:2.0:1787827149.003167:0:6160:0:(obd_config.c:1928:class_config_llog_handler()) skipping 0x6 00000020:01000000:2.0:1787827149.003168:0:6160:0:(obd_config.c:1928:class_config_llog_handler()) skipping 0x6 00000020:01000000:2.0:1787827149.003170:0:6160:0:(obd_config.c:1893:class_config_llog_handler()) Marker, inst_flg=0x6 mark_flg=0x6 00000020:00000080:2.0:1787827149.003173:0:6160:0:(obd_config.c:1472:class_process_config()) processing cmd: cf010 00000020:00000080:2.0:1787827149.003174:0:6160:0:(obd_config.c:1563:class_process_config()) marker 9 (0x6) lustre sys.timeout 00000020:01000000:2.0:1787827149.003177:0:6160:0:(obd_config.c:1893:class_config_llog_handler()) Marker, inst_flg=0x0 mark_flg=0x1 00000020:00000080:2.0:1787827149.003182:0:6160:0:(obd_config.c:1472:class_process_config()) processing cmd: cf010 00000020:00000080:2.0:1787827149.003183:0:6160:0:(obd_config.c:1563:class_process_config()) marker 16 (0x1) lustre-MDT0001 add mdc 00000020:00000080:2.0:1787827149.003187:0:6160:0:(obd_config.c:1472:class_process_config()) processing cmd: cf005 00000020:00000080:2.0:1787827149.003191:0:6160:0:(obd_config.c:1487:class_process_config()) adding mapping from uuid 192.168.203.129@tcp to nid 0x20000c0a8cb81 (192.168.203.129@tcp) 00000020:01000000:2.0:1787827149.003213:0:6160:0:(obd_config.c:2004:class_config_llog_handler()) cmd cf001, instance name: lustre-MDT0001-mdc-ffff8cb98543f000 00000020:00000080:2.0:1787827149.003215:0:6160:0:(obd_config.c:1472:class_process_config()) processing cmd: cf001 00000020:00000080:2.0:1787827149.003224:0:6160:0:(genops.c:413:class_newdev()) Allocate new device lustre-MDT0001-mdc-ffff8cb98543f000 (000000001b434753) 00000020:00000080:2.0:1787827149.003238:0:6160:0:(obd_config.c:720:class_attach_name()) OBD: dev 4 attached type mdc with refcount 2 00000020:01000000:2.0:1787827149.003245:0:6160:0:(obd_config.c:2004:class_config_llog_handler()) cmd cf003, instance name: lustre-MDT0001-mdc-ffff8cb98543f000 00000020:00000080:2.0:1787827149.003247:0:6160:0:(obd_config.c:1472:class_process_config()) processing cmd: cf003 00010000:00080000:2.0:1787827149.003364:0:6160:0:(ldlm_lib.c:102:import_set_conn()) imp 00000000a95ef890@lustre-MDT0001-mdc-ffff8cb98543f000: add connection 192.168.203.129@tcp at head 00010000:00010000:2.0:1787827149.005093:0:6160:0:(ldlm_pool.c:923:ldlm_pool_init()) Lock pool ldlm-pool-lustre-MDT0001-mdc-ffff8cb98543f000-2 is initialized 00000040:01000000:2.0:1787827149.005300:0:6160:0:(llog_obd.c:197:llog_setup()) obd lustre-MDT0001-mdc-ffff8cb98543f000 ctxt 13 is initialized 00000020:00000080:2.0:1787827149.005436:0:6160:0:(obd_config.c:826:class_setup()) finished setup of obd lustre-MDT0001-mdc-ffff8cb98543f000 (uuid 8cef8de3-2326-42de-9d41-f4cb057ed6e8) 00000020:01000000:2.0:1787827149.005445:0:6160:0:(obd_config.c:2004:class_config_llog_handler()) cmd cf014, instance name: lustre-clilmv-ffff8cb98543f000 00000020:00000080:2.0:1787827149.005448:0:6160:0:(obd_config.c:1472:class_process_config()) processing cmd: cf014 00800000:01000000:2.0:1787827149.005460:0:6160:0:(lmv_obd.c:442:lmv_add_target()) Target uuid: lustre-MDT0001_UUID. index 1 00000020:01000000:2.0:1787827149.005504:0:6160:0:(obd_config.c:1893:class_config_llog_handler()) Marker, inst_flg=0x2 mark_flg=0x2 00000020:00000080:2.0:1787827149.005509:0:6160:0:(obd_config.c:1472:class_process_config()) processing cmd: cf010 00000020:00000080:2.0:1787827149.005511:0:6160:0:(obd_config.c:1563:class_process_config()) marker 16 (0x2) lustre-MDT0001 add mdc 00000020:01000000:2.0:1787827149.005514:0:6160:0:(obd_config.c:1893:class_config_llog_handler()) Marker, inst_flg=0x0 mark_flg=0x5 00000020:01000000:2.0:1787827149.005517:0:6160:0:(obd_config.c:1902:class_config_llog_handler()) SKIP #19 00000020:01000000:2.0:1787827149.005518:0:6160:0:(obd_config.c:1928:class_config_llog_handler()) skipping 0x6 00000020:01000000:2.0:1787827149.005520:0:6160:0:(obd_config.c:1928:class_config_llog_handler()) skipping 0x6 00000020:01000000:2.0:1787827149.005522:0:6160:0:(obd_config.c:1893:class_config_llog_handler()) Marker, inst_flg=0x6 mark_flg=0x6 00000020:00000080:2.0:1787827149.005526:0:6160:0:(obd_config.c:1472:class_process_config()) processing cmd: cf010 00000020:00000080:2.0:1787827149.005527:0:6160:0:(obd_config.c:1563:class_process_config()) marker 19 (0x6) lustre sys.timeout 00000020:01000000:2.0:1787827149.005531:0:6160:0:(obd_config.c:1893:class_config_llog_handler()) Marker, inst_flg=0x0 mark_flg=0x1 00000020:01000004:2.0:1787827149.005535:0:6160:0:(obd_mount.c:961:lustre_check_exclusion()) Check exclusion lustre-OST0000 (0) in 0 of 192.168.203.129@tcp:/lustre 00000020:00000080:2.0:1787827149.005540:0:6160:0:(obd_config.c:1472:class_process_config()) processing cmd: cf010 00000020:00000080:2.0:1787827149.005542:0:6160:0:(obd_config.c:1563:class_process_config()) marker 24 (0x1) lustre-OST0000 add osc 00000020:00000080:2.0:1787827149.005546:0:6160:0:(obd_config.c:1472:class_process_config()) processing cmd: cf005 00000020:00000080:2.0:1787827149.005550:0:6160:0:(obd_config.c:1487:class_process_config()) adding mapping from uuid 192.168.203.129@tcp to nid 0x20000c0a8cb81 (192.168.203.129@tcp) 00000020:01000000:2.0:1787827149.005572:0:6160:0:(obd_config.c:2004:class_config_llog_handler()) cmd cf001, instance name: lustre-OST0000-osc-ffff8cb98543f000 00000020:00000080:2.0:1787827149.005576:0:6160:0:(obd_config.c:1472:class_process_config()) processing cmd: cf001 00000020:00000080:2.0:1787827149.005583:0:6160:0:(genops.c:413:class_newdev()) Allocate new device lustre-OST0000-osc-ffff8cb98543f000 (00000000c71278a8) 00000020:00000080:2.0:1787827149.005594:0:6160:0:(obd_config.c:720:class_attach_name()) OBD: dev 5 attached type osc with refcount 2 00000020:01000000:2.0:1787827149.005600:0:6160:0:(obd_config.c:2004:class_config_llog_handler()) cmd cf003, instance name: lustre-OST0000-osc-ffff8cb98543f000 00000020:00000080:2.0:1787827149.005603:0:6160:0:(obd_config.c:1472:class_process_config()) processing cmd: cf003 00010000:00080000:2.0:1787827149.005888:0:6160:0:(ldlm_lib.c:102:import_set_conn()) imp 000000004fe17eaf@lustre-OST0000-osc-ffff8cb98543f000: add connection 192.168.203.129@tcp at head 00010000:00010000:2.0:1787827149.006180:0:6160:0:(ldlm_pool.c:923:ldlm_pool_init()) Lock pool ldlm-pool-lustre-OST0000-osc-ffff8cb98543f000-3 is initialized 00000020:00000080:2.0:1787827149.008447:0:6160:0:(obd_config.c:826:class_setup()) finished setup of obd lustre-OST0000-osc-ffff8cb98543f000 (uuid 8cef8de3-2326-42de-9d41-f4cb057ed6e8) 00000020:01000000:2.0:1787827149.008468:0:6160:0:(obd_config.c:2004:class_config_llog_handler()) cmd cf00d, instance name: lustre-clilov-ffff8cb98543f000 00000020:00000080:2.0:1787827149.008484:0:6160:0:(obd_config.c:1472:class_process_config()) processing cmd: cf00d 00020000:01000000:2.0:1787827149.008505:0:6160:0:(lov_obd.c:481:lov_add_target()) uuid:lustre-OST0000_UUID idx:0 gen:1 active:1 00020000:01000000:2.0:1787827149.008513:0:6160:0:(lov_obd.c:510:lov_add_target()) lustre-clilov-ffff8cb98543f000: Adding target lustre-OST0000_UUID 00020000:01000000:2.0:1787827149.008518:0:6160:0:(lov_obd.c:532:lov_add_target()) idx=0 ltd_gen=1 ld_tgt_count=1 00000020:01000000:2.0:1787827149.008532:0:6160:0:(obd_config.c:1893:class_config_llog_handler()) Marker, inst_flg=0x2 mark_flg=0x2 00000020:00000080:2.0:1787827149.008536:0:6160:0:(obd_config.c:1472:class_process_config()) processing cmd: cf010 00000020:00000080:2.0:1787827149.008538:0:6160:0:(obd_config.c:1563:class_process_config()) marker 24 (0x2) lustre-OST0000 add osc 00000020:01000000:2.0:1787827149.008542:0:6160:0:(obd_config.c:1893:class_config_llog_handler()) Marker, inst_flg=0x0 mark_flg=0x5 00000020:01000000:2.0:1787827149.008545:0:6160:0:(obd_config.c:1902:class_config_llog_handler()) SKIP #27 00000020:01000000:2.0:1787827149.008547:0:6160:0:(obd_config.c:1928:class_config_llog_handler()) skipping 0x6 00000020:01000000:2.0:1787827149.008549:0:6160:0:(obd_config.c:1928:class_config_llog_handler()) skipping 0x6 00000020:01000000:2.0:1787827149.008551:0:6160:0:(obd_config.c:1893:class_config_llog_handler()) Marker, inst_flg=0x6 mark_flg=0x6 00000020:00000080:2.0:1787827149.008554:0:6160:0:(obd_config.c:1472:class_process_config()) processing cmd: cf010 00000020:00000080:2.0:1787827149.008555:0:6160:0:(obd_config.c:1563:class_process_config()) marker 27 (0x6) lustre sys.timeout 00000020:01000000:2.0:1787827149.008558:0:6160:0:(obd_config.c:1893:class_config_llog_handler()) Marker, inst_flg=0x0 mark_flg=0x1 00000020:01000004:2.0:1787827149.008562:0:6160:0:(obd_mount.c:961:lustre_check_exclusion()) Check exclusion lustre-OST0001 (1) in 0 of 192.168.203.129@tcp:/lustre 00000020:00000080:2.0:1787827149.008566:0:6160:0:(obd_config.c:1472:class_process_config()) processing cmd: cf010 00000020:00000080:2.0:1787827149.008568:0:6160:0:(obd_config.c:1563:class_process_config()) marker 32 (0x1) lustre-OST0001 add osc 00000020:00000080:2.0:1787827149.008572:0:6160:0:(obd_config.c:1472:class_process_config()) processing cmd: cf005 00000020:00000080:2.0:1787827149.008577:0:6160:0:(obd_config.c:1487:class_process_config()) adding mapping from uuid 192.168.203.129@tcp to nid 0x20000c0a8cb81 (192.168.203.129@tcp) 00000020:01000000:2.0:1787827149.008600:0:6160:0:(obd_config.c:2004:class_config_llog_handler()) cmd cf001, instance name: lustre-OST0001-osc-ffff8cb98543f000 00000020:00000080:2.0:1787827149.008602:0:6160:0:(obd_config.c:1472:class_process_config()) processing cmd: cf001 00000020:00000080:2.0:1787827149.008610:0:6160:0:(genops.c:413:class_newdev()) Allocate new device lustre-OST0001-osc-ffff8cb98543f000 (000000007416da0a) 00000020:00000080:2.0:1787827149.008625:0:6160:0:(obd_config.c:720:class_attach_name()) OBD: dev 6 attached type osc with refcount 2 00000020:01000000:2.0:1787827149.008631:0:6160:0:(obd_config.c:2004:class_config_llog_handler()) cmd cf003, instance name: lustre-OST0001-osc-ffff8cb98543f000 00000020:00000080:2.0:1787827149.008634:0:6160:0:(obd_config.c:1472:class_process_config()) processing cmd: cf003 00010000:00080000:2.0:1787827149.008717:0:6160:0:(ldlm_lib.c:102:import_set_conn()) imp 000000001a45ae89@lustre-OST0001-osc-ffff8cb98543f000: add connection 192.168.203.129@tcp at head 00010000:00010000:2.0:1787827149.008988:0:6160:0:(ldlm_pool.c:923:ldlm_pool_init()) Lock pool ldlm-pool-lustre-OST0001-osc-ffff8cb98543f000-4 is initialized 00000020:00000080:2.0:1787827149.009286:0:6160:0:(obd_config.c:826:class_setup()) finished setup of obd lustre-OST0001-osc-ffff8cb98543f000 (uuid 8cef8de3-2326-42de-9d41-f4cb057ed6e8) 00000020:01000000:2.0:1787827149.009295:0:6160:0:(obd_config.c:2004:class_config_llog_handler()) cmd cf00d, instance name: lustre-clilov-ffff8cb98543f000 00000020:00000080:2.0:1787827149.009304:0:6160:0:(obd_config.c:1472:class_process_config()) processing cmd: cf00d 00020000:01000000:2.0:1787827149.009316:0:6160:0:(lov_obd.c:481:lov_add_target()) uuid:lustre-OST0001_UUID idx:1 gen:1 active:1 00020000:01000000:2.0:1787827149.009321:0:6160:0:(lov_obd.c:510:lov_add_target()) lustre-clilov-ffff8cb98543f000: Adding target lustre-OST0001_UUID 00020000:01000000:2.0:1787827149.009323:0:6160:0:(lov_obd.c:532:lov_add_target()) idx=1 ltd_gen=1 ld_tgt_count=2 00000020:01000000:2.0:1787827149.009333:0:6160:0:(obd_config.c:1893:class_config_llog_handler()) Marker, inst_flg=0x2 mark_flg=0x2 00000020:00000080:2.0:1787827149.009345:0:6160:0:(obd_config.c:1472:class_process_config()) processing cmd: cf010 00000020:00000080:2.0:1787827149.009347:0:6160:0:(obd_config.c:1563:class_process_config()) marker 32 (0x2) lustre-OST0001 add osc 00000020:01000000:2.0:1787827149.009350:0:6160:0:(obd_config.c:1893:class_config_llog_handler()) Marker, inst_flg=0x0 mark_flg=0x1 00000020:00000080:2.0:1787827149.009354:0:6160:0:(obd_config.c:1472:class_process_config()) processing cmd: cf010 00000020:00000080:2.0:1787827149.009356:0:6160:0:(obd_config.c:1563:class_process_config()) marker 36 (0x1) lustre sys.timeout 00000020:00000080:2.0:1787827149.009359:0:6160:0:(obd_config.c:1472:class_process_config()) processing cmd: cf009 00000020:00000080:2.0:1787827149.009360:0:6160:0:(obd_config.c:1537:class_process_config()) changing lustre timeout from 100 to 20 00000100:00100000:2.0:1787827149.009397:0:6160:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@00000000817ec53a pname:cluuid:pid:xid:nid:opc:job llog_process_th:0bb8b105-f59c-455a-b6b1-ce40efc3999c:6160:1874672572433408:192.168.203.129@tcp:502: 00000100:00100000:2.0:1787827149.009524:0:6160:0:(client.c:2715:ptlrpc_set_wait()) set 00000000b7107a21 going to sleep for 11 seconds 00000100:00100000:2.0:1787827149.011772:0:6160:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@00000000817ec53a pname:cluuid:pid:xid:nid:opc:job llog_process_th:0bb8b105-f59c-455a-b6b1-ce40efc3999c:6160:1874672572433408:192.168.203.129@tcp:502: 00000020:01000000:2.0:1787827149.011810:0:6160:0:(obd_config.c:1893:class_config_llog_handler()) Marker, inst_flg=0x2 mark_flg=0x2 00000020:00000080:2.0:1787827149.011816:0:6160:0:(obd_config.c:1472:class_process_config()) processing cmd: cf010 00000020:00000080:2.0:1787827149.011818:0:6160:0:(obd_config.c:1563:class_process_config()) marker 36 (0x2) lustre sys.timeout 00000040:00080000:2.0:1787827149.011822:0:6160:0:(llog.c:862:llog_process_thread()) stop processing plain [0xa:0x3:0x0] index 48 count 48 00000020:01000000:2.0:1787827149.011889:0:6141:0:(obd_config.c:2145:class_config_parse_llog()) Processed log lustre-client gen 1-47 (rc=0) 10000000:01000000:2.0:1787827149.011905:0:6141:0:(mgc_request.c:1879:mgc_process_log()) MGC192.168.203.129@tcp: configuration from log 'lustre-client' succeeded (0). 00010000:00010000:2.0:1787827149.011916:0:6141:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: MGC192.168.203.129@tcp lock: 000000005a726d24/0x218c3ba1346961b lrc: 3/1,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x11000000000000 nid: local remote: 0xb2e8bd65d71dd9f7 expref: -99 pid: 6141 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1787827149.011924:0:6141:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: MGC192.168.203.129@tcp lock: 000000005a726d24/0x218c3ba1346961b lrc: 2/0,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x11000000000000 nid: local remote: 0xb2e8bd65d71dd9f7 expref: -99 pid: 6141 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:01000000:2.0:1787827149.011931:0:6141:0:(mgc_request.c:1802:mgc_process_log()) Process log lustre-cliir-ffff8cb98543f000 from 1 10000000:01000000:2.0:1787827149.011934:0:6141:0:(mgc_request.c:996:mgc_enqueue()) Enqueue for lustre-cliir (res 0x65727473756c) 00010000:00010000:2.0:1787827149.011963:0:6141:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: MGC192.168.203.129@tcp lock: 0000000029fef930/0x218c3ba1346964c lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 6141 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1787827149.011976:0:6141:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000000000000 ns: MGC192.168.203.129@tcp lock: 0000000029fef930/0x218c3ba1346964c lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 6141 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1787827149.011986:0:6141:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: MGC192.168.203.129@tcp lock: 0000000029fef930/0x218c3ba1346964c lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 6141 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1787827149.011998:0:6141:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@00000000817ec53a pname:cluuid:pid:xid:nid:opc:job mount.lustre:0bb8b105-f59c-455a-b6b1-ce40efc3999c:6141:1874672572433536:192.168.203.129@tcp:101: 00000100:00100000:2.0:1787827149.012043:0:6141:0:(client.c:2715:ptlrpc_set_wait()) set 00000000b7107a21 going to sleep for 11 seconds 00000100:00100000:3.0:1787827149.013162:0:6141:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@00000000817ec53a pname:cluuid:pid:xid:nid:opc:job mount.lustre:0bb8b105-f59c-455a-b6b1-ce40efc3999c:6141:1874672572433536:192.168.203.129@tcp:101: 00010000:00010000:3.0:1787827149.013180:0:6141:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: MGC192.168.203.129@tcp lock: 0000000029fef930/0x218c3ba1346964c lrc: 4/1,0 mode: CR/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x11000000000000 nid: local remote: 0xb2e8bd65d71dd9fe expref: -99 pid: 6141 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787827149.013189:0:6141:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: MGC192.168.203.129@tcp lock: 0000000029fef930/0x218c3ba1346964c lrc: 4/1,0 mode: CR/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0xb2e8bd65d71dd9fe expref: -99 pid: 6141 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1787827149.013794:0:6141:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@000000002fce079b pname:cluuid:pid:xid:nid:opc:job mount.lustre:0bb8b105-f59c-455a-b6b1-ce40efc3999c:6141:1874672572433664:192.168.203.129@tcp:256: 00000100:00100000:3.0:1787827149.013852:0:6141:0:(client.c:2715:ptlrpc_set_wait()) set 0000000096ca493e going to sleep for 11 seconds 00000100:00100000:3.0:1787827149.016681:0:6141:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@000000002fce079b pname:cluuid:pid:xid:nid:opc:job mount.lustre:0bb8b105-f59c-455a-b6b1-ce40efc3999c:6141:1874672572433664:192.168.203.129@tcp:256: 10000000:01000000:3.0:1787827149.017242:0:6141:0:(mgc_request.c:1879:mgc_process_log()) MGC192.168.203.129@tcp: configuration from log 'lustre-cliir' succeeded (0). 00010000:00010000:3.0:1787827149.017276:0:6141:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: MGC192.168.203.129@tcp lock: 0000000029fef930/0x218c3ba1346964c lrc: 3/1,0 mode: CR/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x11000000000000 nid: local remote: 0xb2e8bd65d71dd9fe expref: -99 pid: 6141 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787827149.017286:0:6141:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: MGC192.168.203.129@tcp lock: 0000000029fef930/0x218c3ba1346964c lrc: 2/0,0 mode: CR/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x11000000000000 nid: local remote: 0xb2e8bd65d71dd9fe expref: -99 pid: 6141 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:01000000:3.0:1787827149.017294:0:6141:0:(mgc_request.c:1802:mgc_process_log()) Process log params-ffff8cb98543f000 from 1 10000000:01000000:3.0:1787827149.017298:0:6141:0:(mgc_request.c:996:mgc_enqueue()) Enqueue for params (res 0x736d61726170) 00010000:00010000:3.0:1787827149.017323:0:6141:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: MGC192.168.203.129@tcp lock: 0000000066c42af6/0x218c3ba13469653 lrc: 3/1,0 mode: --/CR res: [0x736d61726170:0x3:0x0].0x0 rrc: 2 type: PLN flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 6141 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787827149.017330:0:6141:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000000000000 ns: MGC192.168.203.129@tcp lock: 0000000066c42af6/0x218c3ba13469653 lrc: 3/1,0 mode: --/CR res: [0x736d61726170:0x3:0x0].0x0 rrc: 2 type: PLN flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 6141 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787827149.017337:0:6141:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: MGC192.168.203.129@tcp lock: 0000000066c42af6/0x218c3ba13469653 lrc: 3/1,0 mode: --/CR res: [0x736d61726170:0x3:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 6141 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1787827149.017352:0:6141:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@000000002fce079b pname:cluuid:pid:xid:nid:opc:job mount.lustre:0bb8b105-f59c-455a-b6b1-ce40efc3999c:6141:1874672572433792:192.168.203.129@tcp:101: 00000100:00100000:3.0:1787827149.017411:0:6141:0:(client.c:2715:ptlrpc_set_wait()) set 0000000096ca493e going to sleep for 11 seconds 00000100:00100000:3.0:1787827149.018288:0:6141:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@000000002fce079b pname:cluuid:pid:xid:nid:opc:job mount.lustre:0bb8b105-f59c-455a-b6b1-ce40efc3999c:6141:1874672572433792:192.168.203.129@tcp:101: 00010000:00010000:3.0:1787827149.018306:0:6141:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: MGC192.168.203.129@tcp lock: 0000000066c42af6/0x218c3ba13469653 lrc: 4/1,0 mode: CR/CR res: [0x736d61726170:0x3:0x0].0x0 rrc: 2 type: PLN flags: 0x11000000000000 nid: local remote: 0xb2e8bd65d71dda05 expref: -99 pid: 6141 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787827149.018328:0:6141:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: MGC192.168.203.129@tcp lock: 0000000066c42af6/0x218c3ba13469653 lrc: 4/1,0 mode: CR/CR res: [0x736d61726170:0x3:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0xb2e8bd65d71dda05 expref: -99 pid: 6141 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1787827149.018559:0:6141:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@00000000817ec53a pname:cluuid:pid:xid:nid:opc:job mount.lustre:0bb8b105-f59c-455a-b6b1-ce40efc3999c:6141:1874672572433920:192.168.203.129@tcp:501: 00000100:00100000:2.0:1787827149.018598:0:6141:0:(client.c:2715:ptlrpc_set_wait()) set 000000003b8fc3aa going to sleep for 11 seconds 00000100:00100000:2.0:1787827149.019731:0:6141:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@00000000817ec53a pname:cluuid:pid:xid:nid:opc:job mount.lustre:0bb8b105-f59c-455a-b6b1-ce40efc3999c:6141:1874672572433920:192.168.203.129@tcp:501: 00000100:00100000:2.0:1787827149.019762:0:6141:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@00000000817ec53a pname:cluuid:pid:xid:nid:opc:job mount.lustre:0bb8b105-f59c-455a-b6b1-ce40efc3999c:6141:1874672572434048:192.168.203.129@tcp:503: 00000100:00100000:2.0:1787827149.019807:0:6141:0:(client.c:2715:ptlrpc_set_wait()) set 000000003b8fc3aa going to sleep for 11 seconds 00000100:00100000:2.0:1787827149.020686:0:6141:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@00000000817ec53a pname:cluuid:pid:xid:nid:opc:job mount.lustre:0bb8b105-f59c-455a-b6b1-ce40efc3999c:6141:1874672572434048:192.168.203.129@tcp:503: 00000100:00100000:2.0:1787827149.020927:0:6162:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@00000000817ec53a pname:cluuid:pid:xid:nid:opc:job llog_process_th:0bb8b105-f59c-455a-b6b1-ce40efc3999c:6162:1874672572434176:192.168.203.129@tcp:502: 00000100:00100000:2.0:1787827149.020976:0:6162:0:(client.c:2715:ptlrpc_set_wait()) set 0000000073328c92 going to sleep for 11 seconds 00000100:00100000:2.0:1787827149.021877:0:6162:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@00000000817ec53a pname:cluuid:pid:xid:nid:opc:job llog_process_th:0bb8b105-f59c-455a-b6b1-ce40efc3999c:6162:1874672572434176:192.168.203.129@tcp:502: 00000020:01000000:2.0:1787827149.021906:0:6162:0:(obd_config.c:1893:class_config_llog_handler()) Marker, inst_flg=0x0 mark_flg=0x1 00000020:00000080:2.0:1787827149.021915:0:6162:0:(obd_config.c:1472:class_process_config()) processing cmd: cf010 00000020:00000080:2.0:1787827149.021917:0:6162:0:(obd_config.c:1563:class_process_config()) marker 2 (0x1) general debug_raw_pointers 00000020:01000000:2.0:1787827149.021922:0:6162:0:(obd_config.c:2004:class_config_llog_handler()) cmd ce032, instance name: general-ffff8cb98543f000 00000020:00000080:2.0:1787827149.021925:0:6162:0:(obd_config.c:1472:class_process_config()) processing cmd: ce032 00000100:00100000:2.0:1787827149.021965:0:6162:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f7f480 pname:cluuid:pid:xid:nid:opc:job llog_process_th:0bb8b105-f59c-455a-b6b1-ce40efc3999c:6162:1874672572434304:192.168.203.129@tcp:502: 00000100:00100000:2.0:1787827149.022018:0:6162:0:(client.c:2715:ptlrpc_set_wait()) set ffff8cb988d76540 going to sleep for 11 seconds 00000100:00100000:2.0:1787827149.023009:0:6162:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f7f480 pname:cluuid:pid:xid:nid:opc:job llog_process_th:0bb8b105-f59c-455a-b6b1-ce40efc3999c:6162:1874672572434304:192.168.203.129@tcp:502: 00000020:01000000:2.0:1787827149.023033:0:6162:0:(obd_config.c:1893:class_config_llog_handler()) Marker, inst_flg=0x2 mark_flg=0x2 00000020:00000080:2.0:1787827149.023038:0:6162:0:(obd_config.c:1472:class_process_config()) processing cmd: cf010 00000020:00000080:2.0:1787827149.023039:0:6162:0:(obd_config.c:1563:class_process_config()) marker 2 (0x2) general debug_raw_pointers 00000040:00080000:2.0:1787827149.023042:0:6162:0:(llog.c:862:llog_process_thread()) stop processing plain [0xa:0x2:0x0] index 4 count 4 00000020:01000000:0.0:1787827149.023112:0:6141:0:(obd_config.c:2145:class_config_parse_llog()) Processed log params gen 1-3 (rc=0) 10000000:01000000:0.0:1787827149.023129:0:6141:0:(mgc_request.c:1879:mgc_process_log()) MGC192.168.203.129@tcp: configuration from log 'params' succeeded (0). 00010000:00010000:0.0:1787827149.023150:0:6141:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: MGC192.168.203.129@tcp lock: ffff8cb981674800/0x218c3ba13469653 lrc: 3/1,0 mode: CR/CR res: [0x736d61726170:0x3:0x0].0x0 rrc: 2 type: PLN flags: 0x11000000000000 nid: local remote: 0xb2e8bd65d71dda05 expref: -99 pid: 6141 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787827149.023162:0:6141:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: MGC192.168.203.129@tcp lock: ffff8cb981674800/0x218c3ba13469653 lrc: 2/0,0 mode: CR/CR res: [0x736d61726170:0x3:0x0].0x0 rrc: 2 type: PLN flags: 0x11000000000000 nid: local remote: 0xb2e8bd65d71dda05 expref: -99 pid: 6141 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000080:01000000:0.0:1787827149.023184:0:6141:0:(llite_lib.c:1614:ll_fill_super()) Found profile lustre-client: mdc=lustre-clilmv osc=lustre-clilov 00000020:00000080:0.0:1787827149.023205:0:6141:0:(genops.c:1385:class_connect()) connect: client 8cef8de3-2326-42de-9d41-f4cb057ed6e8, cookie 0x218c3ba1346965a 00800000:01000000:0.0:1787827149.023225:0:6141:0:(lmv_obd.c:515:lmv_check_connect()) Time to connect 8cef8de3-2326-42de-9d41-f4cb057ed6e8 to lustre-clilmv-ffff8cb98543f000 00800000:01000000:0.0:1787827149.023231:0:6141:0:(lmv_obd.c:338:lmv_connect_mdc()) connect to lustre-MDT0000-mdc-ffff8cb98543f000(8cef8de3-2326-42de-9d41-f4cb057ed6e8) - lustre-MDT0000_UUID, 8cef8de3-2326-42de-9d41-f4cb057ed6e8 00000020:00000080:0.0:1787827149.023238:0:6141:0:(genops.c:1385:class_connect()) connect: client 8cef8de3-2326-42de-9d41-f4cb057ed6e8, cookie 0x218c3ba13469661 00000100:00080000:0.0:1787827149.023246:0:6141:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 0 00000100:00080000:0.0:1787827149.023253:0:6141:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:0.0:1787827149.023305:0:6141:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 134217728, lost: 0. 00000100:00100000:0.0:1787827149.023325:0:6141:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c68b40 x1874672572434432/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787827149.023344:0:6141:0:(pinger.c:373:ptlrpc_pinger_add_import()) adding pingable import 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID 00000100:00100000:2.0:1787827149.023368:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c68b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572434432:192.168.203.129@tcp:38: 00800000:01000000:0.0:1787827149.023432:0:6141:0:(lmv_obd.c:392:lmv_connect_mdc()) Connected to lustre-MDT0000-mdc-ffff8cb98543f000(8cef8de3-2326-42de-9d41-f4cb057ed6e8) successfully (5) 00800000:00000004:0.0:1787827149.023437:0:6141:0:(obd_class.h:1056:obd_statfs_async()) lustre-MDT0000-mdc-ffff8cb98543f000: age -894, max_age 0 00800000:01000000:0.0:1787827149.023464:0:6141:0:(lmv_obd.c:338:lmv_connect_mdc()) connect to lustre-MDT0001-mdc-ffff8cb98543f000(8cef8de3-2326-42de-9d41-f4cb057ed6e8) - lustre-MDT0001_UUID, 8cef8de3-2326-42de-9d41-f4cb057ed6e8 00000020:00000080:0.0:1787827149.023486:0:6141:0:(genops.c:1385:class_connect()) connect: client 8cef8de3-2326-42de-9d41-f4cb057ed6e8, cookie 0x218c3ba13469668 00000100:00080000:0.0:1787827149.023498:0:6141:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 0 00000100:00080000:0.0:1787827149.023507:0:6141:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:0.0:1787827149.023513:0:6141:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 134217728, lost: 0. 00000100:00100000:0.0:1787827149.023527:0:6141:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c6a940 x1874672572434688/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787827149.023542:0:6141:0:(pinger.c:373:ptlrpc_pinger_add_import()) adding pingable import 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID 00000100:00100000:2.0:1787827149.023575:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c6a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572434688:192.168.203.129@tcp:38: 00000100:00080000:1.0F:1787827149.023596:0:2395:0:(client.c:1845:ptlrpc_send_new_req()) @@@ req waiting for recovery: (FULL != CONNECTING) req@ffff8cb999c68000 x1874672572434560/t0(0) o41->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 224/368 e 0 to 0 dl 0 ref 1 fl Rpc:WQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00800000:01000000:0.0:1787827149.023605:0:6141:0:(lmv_obd.c:392:lmv_connect_mdc()) Connected to lustre-MDT0001-mdc-ffff8cb98543f000(8cef8de3-2326-42de-9d41-f4cb057ed6e8) successfully (5) 00800000:00000004:0.0:1787827149.023610:0:6141:0:(obd_class.h:1056:obd_statfs_async()) lustre-MDT0001-mdc-ffff8cb98543f000: age -894, max_age 0 00000100:00100000:1.0:1787827149.023626:0:2395:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000080:00000004:0.0:1787827149.023635:0:6141:0:(obd_class.h:1090:obd_statfs()) lustre-clilmv-ffff8cb98543f000: age -894, max_age 105 00000100:00080000:1.0:1787827149.023636:0:2395:0:(client.c:1845:ptlrpc_send_new_req()) @@@ req waiting for recovery: (FULL != CONNECTING) req@ffff8cb999c6b480 x1874672572434816/t0(0) o41->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 224/368 e 0 to 0 dl 0 ref 1 fl Rpc:WQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00800000:00000004:0.0:1787827149.023643:0:6141:0:(obd_class.h:1090:obd_statfs()) lustre-MDT0000-mdc-ffff8cb98543f000: age -894, max_age 105 00000100:00080000:0.0:1787827149.023660:0:6141:0:(client.c:1845:ptlrpc_send_new_req()) @@@ req waiting for recovery: (FULL != CONNECTING) req@ffff8cb999c6a580 x1874672572434944/t0(0) o41->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 224/368 e 0 to 0 dl 0 ref 2 fl Rpc:WQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827149.023668:0:6141:0:(client.c:2715:ptlrpc_set_wait()) set ffff8cb988bda840 going to sleep for 0 seconds 00010000:00080000:2.0:1787827149.024581:0:2393:0:(ldlm_request.c:1671:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=46162) req@ffff8cb999c68b40 x1874672572434432/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/416 e 0 to 0 dl 1787827159 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787827149.024589:0:2393:0:(client.c:3125:ptlrpc_free_committed()) lustre-MDT0000-mdc-ffff8cb98543f000: committing for last_committed 0 gen 1 00000100:00080000:2.0:1787827149.024594:0:2393:0:(import.c:1132:ptlrpc_connect_interpret()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to target with instance 1 00000001:01000000:2.0:1787827149.274358:0:2393:0:(integrity.c:236:obd_t10_performance_test()) lustre-MDT0000-mdc-ffff8cb98543f000: T10 checksum algorithm t10ip512 speed = 6643 MB/s 00000001:01000000:2.0:1787827149.524283:0:2393:0:(integrity.c:236:obd_t10_performance_test()) lustre-MDT0000-mdc-ffff8cb98543f000: T10 checksum algorithm t10ip4K speed = 7959 MB/s 00000001:01000000:2.0:1787827149.774748:0:2393:0:(integrity.c:236:obd_t10_performance_test()) lustre-MDT0000-mdc-ffff8cb98543f000: T10 checksum algorithm t10crc512 speed = 1427 MB/s 00010000:00010000:3.0:1787827150.019698:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000001:01000000:2.0:1787827150.024682:0:2393:0:(integrity.c:236:obd_t10_performance_test()) lustre-MDT0000-mdc-ffff8cb98543f000: T10 checksum algorithm t10crc4K speed = 1719 MB/s 00000100:00080000:2.0:1787827150.024695:0:2393:0:(import.c:956:ptlrpc_connect_set_flags()) lustre-MDT0000-mdc-ffff8cb98543f000: Resetting ns_connect_flags to server flags: 0xae7a7e7be344d3b8 00000100:00080000:2.0:1787827150.024708:0:2393:0:(import.c:1213:ptlrpc_connect_interpret()) connected to replayable target: lustre-MDT0000_UUID 00000100:00080000:2.0:1787827150.024712:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to FULL 00000100:00080000:0.0:1787827150.025076:0:2393:0:(recover.c:191:ptlrpc_wake_delayed()) @@@ waking (set 000000006978a037): req@ffff8cb999c68000 x1874672572434560/t0(0) o41->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 224/368 e 0 to 0 dl 0 ref 1 fl Rpc:WQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827150.025102:0:2393:0:(recover.c:191:ptlrpc_wake_delayed()) @@@ waking (set 0000000069b786f2): req@ffff8cb999c6a580 x1874672572434944/t0(0) o41->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 224/368 e 0 to 0 dl 0 ref 2 fl Rpc:WQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827150.025193:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c68b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572434432:192.168.203.129@tcp:38: 00010000:00080000:0.0:1787827150.025246:0:2393:0:(ldlm_request.c:1671:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=46162) req@ffff8cb999c6a940 x1874672572434688/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/416 e 0 to 0 dl 1787827159 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827150.025257:0:2393:0:(client.c:3125:ptlrpc_free_committed()) lustre-MDT0001-mdc-ffff8cb98543f000: committing for last_committed 0 gen 1 00000100:00080000:0.0:1787827150.025293:0:2393:0:(import.c:1132:ptlrpc_connect_interpret()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to target with instance 1 00000100:00080000:0.0:1787827150.025301:0:2393:0:(import.c:956:ptlrpc_connect_set_flags()) lustre-MDT0001-mdc-ffff8cb98543f000: Resetting ns_connect_flags to server flags: 0xae7a7e7be344d3b8 00000100:00080000:0.0:1787827150.025307:0:2393:0:(import.c:1213:ptlrpc_connect_interpret()) connected to replayable target: lustre-MDT0001_UUID 00000100:00080000:0.0:1787827150.025309:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to FULL 00000100:00100000:3.0:1787827150.025365:0:6141:0:(client.c:2715:ptlrpc_set_wait()) set ffff8cb988bda840 going to sleep for 0 seconds 00000100:00100000:3.0:1787827150.025511:0:6141:0:(client.c:2715:ptlrpc_set_wait()) set ffff8cb988bda840 going to sleep for 11 seconds 00000100:00100000:0.0:1787827150.026013:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c6a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572434688:192.168.203.129@tcp:38: 00010000:00080000:1.0:1787827150.027699:0:2395:0:(ldlm_request.c:1671:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=46162) req@ffff8cb999c6b480 x1874672572434816/t0(0) o41->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 224/368 e 0 to 0 dl 1787827166 ref 1 fl Rpc:RQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787827150.027713:0:2395:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c6b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:8cef8de3-2326-42de-9d41-f4cb057ed6e8:0:1874672572434816:192.168.203.129@tcp:41: 00010000:00080000:1.0:1787827150.027719:0:2395:0:(ldlm_request.c:1671:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=46162) req@ffff8cb999c68000 x1874672572434560/t0(0) o41->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 224/368 e 0 to 0 dl 1787827166 ref 1 fl Rpc:RQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787827150.027725:0:2395:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c68000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:8cef8de3-2326-42de-9d41-f4cb057ed6e8:0:1874672572434560:192.168.203.129@tcp:41: 00010000:00080000:3.0:1787827150.027986:0:6141:0:(ldlm_request.c:1671:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=46162) req@ffff8cb999c6a580 x1874672572434944/t0(0) o41->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 224/368 e 0 to 0 dl 1787827166 ref 2 fl Rpc:RQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827150.027999:0:6141:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c6a580 pname:cluuid:pid:xid:nid:opc:job mount.lustre:8cef8de3-2326-42de-9d41-f4cb057ed6e8:6141:1874672572434944:192.168.203.129@tcp:41: 00800000:00000004:3.0:1787827150.028012:0:6141:0:(obd_class.h:1116:obd_statfs()) lustre-MDT0000-mdc-ffff8cb98543f000: update ffff8cb998a3d178 cache blocks 317411/353529 objects 1023725/1024000 00000080:00000004:3.0:1787827150.028018:0:6141:0:(obd_class.h:1116:obd_statfs()) lustre-clilmv-ffff8cb98543f000: update ffff8cb998a38450 cache blocks 317411/353529 objects 1023725/1024000 00000100:00100000:3.0:1787827150.028048:0:6141:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb992314780 pname:cluuid:pid:xid:nid:opc:job mount.lustre:8cef8de3-2326-42de-9d41-f4cb057ed6e8:6141:1874672572435072:192.168.203.129@tcp:40: 00000100:00100000:3.0:1787827150.028100:0:6141:0:(client.c:2715:ptlrpc_set_wait()) set ffff8cb988c06480 going to sleep for 11 seconds 00010000:00080000:3.0:1787827150.030701:0:6141:0:(ldlm_request.c:1671:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=46162) req@ffff8cb992314780 x1874672572435072/t0(0) o40->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 440/448 e 0 to 0 dl 1787827166 ref 2 fl Rpc:RQU/209/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827150.030717:0:6141:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb992314780 pname:cluuid:pid:xid:nid:opc:job mount.lustre:8cef8de3-2326-42de-9d41-f4cb057ed6e8:6141:1874672572435072:192.168.203.129@tcp:40: 00000080:00000004:3.0:1787827150.030738:0:6141:0:(llite_lib.c:614:client_common_fill_super()) rootfid [0x200000007:0x1:0x0] 00000080:00100000:3.0:1787827150.030750:0:6141:0:(llite_lib.c:775:client_common_fill_super()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683201 ocd_grant: 2146304 00020000:01000000:3.0:1787827150.030754:0:6141:0:(lov_obd.c:186:lov_connect()) connect #0 00000020:00000080:3.0:1787827150.030765:0:6141:0:(genops.c:1385:class_connect()) connect: client 8cef8de3-2326-42de-9d41-f4cb057ed6e8, cookie 0x218c3ba1346966f 00000020:00000080:3.0:1787827150.030782:0:6141:0:(genops.c:1385:class_connect()) connect: client LOV_OSC_UUID, cookie 0x218c3ba13469676 00000100:00080000:3.0:1787827150.030789:0:6141:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 0 00000100:00080000:3.0:1787827150.030796:0:6141:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8cb98543f000: import ffff8cb998ab1000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:3.0:1787827150.030808:0:6141:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683201 ocd_grant: 134217728, lost: 0. 00000100:00100000:3.0:1787827150.030838:0:6141:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb992314780 x1874672572435200/t0(0) o8->lustre-OST0000-osc-ffff8cb98543f000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787827150.030858:0:6141:0:(pinger.c:373:ptlrpc_pinger_add_import()) adding pingable import 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-OST0000_UUID 00020000:01000000:3.0:1787827150.030882:0:6141:0:(lov_obd.c:159:lov_connect_osc()) Connected tgt idx 0 lustre-OST0000_UUID (lustre-OST0000-osc-ffff8cb98543f000) active 00000100:00100000:0.0:1787827150.030897:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb992314780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572435200:192.168.203.129@tcp:8: 00000020:00000080:3.0:1787827150.030907:0:6141:0:(genops.c:1385:class_connect()) connect: client LOV_OSC_UUID, cookie 0x218c3ba1346967d 00000100:00080000:3.0:1787827150.030914:0:6141:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 0 00000100:00080000:3.0:1787827150.030919:0:6141:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8cb98543f000: import ffff8cb998ab7800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:3.0:1787827150.030925:0:6141:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683201 ocd_grant: 134217728, lost: 0. 00000100:00100000:3.0:1787827150.030937:0:6141:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb992317480 x1874672572435328/t0(0) o8->lustre-OST0001-osc-ffff8cb98543f000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787827150.030946:0:6141:0:(pinger.c:373:ptlrpc_pinger_add_import()) adding pingable import 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-OST0001_UUID 00020000:01000000:3.0:1787827150.030952:0:6141:0:(lov_obd.c:159:lov_connect_osc()) Connected tgt idx 1 lustre-OST0001_UUID (lustre-OST0001-osc-ffff8cb98543f000) active 00000100:00100000:0.0:1787827150.030985:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb992317480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572435328:192.168.203.129@tcp:8: 00000100:00100000:3.0:1787827150.030987:0:6141:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb992314b40 pname:cluuid:pid:xid:nid:opc:job mount.lustre:8cef8de3-2326-42de-9d41-f4cb057ed6e8:6141:1874672572435456:192.168.203.129@tcp:900: 00000100:00100000:3.0:1787827150.031031:0:6141:0:(client.c:2715:ptlrpc_set_wait()) set ffff8cb988c06480 going to sleep for 11 seconds 00000100:00100000:2.0:1787827150.031304:0:2396:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:2.0:1787827150.031314:0:2396:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f7f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2396:1874672572435584:192.168.203.129@tcp:400: 00010000:00080000:3.0:1787827150.032887:0:6141:0:(ldlm_request.c:1671:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=46162) req@ffff8cb992314b40 x1874672572435456/t0(0) o900->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:29/10 lens 264/248 e 0 to 0 dl 1787827166 ref 2 fl Rpc:RQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827150.032903:0:6141:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb992314b40 pname:cluuid:pid:xid:nid:opc:job mount.lustre:8cef8de3-2326-42de-9d41-f4cb057ed6e8:6141:1874672572435456:192.168.203.129@tcp:900: 00000100:00100000:3.0:1787827150.032947:0:6141:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb992314b40 pname:cluuid:pid:xid:nid:opc:job mount.lustre:8cef8de3-2326-42de-9d41-f4cb057ed6e8:6141:1874672572435712:192.168.203.129@tcp:33: 00000100:00100000:3.0:1787827150.033129:0:6141:0:(client.c:2715:ptlrpc_set_wait()) set ffff8cb988c06480 going to sleep for 11 seconds 00010000:00080000:2.0:1787827150.033563:0:2393:0:(ldlm_request.c:1671:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=46162) req@ffff8cb992317480 x1874672572435328/t0(0) o8->lustre-OST0001-osc-ffff8cb98543f000@192.168.203.129@tcp:28/4 lens 520/416 e 0 to 0 dl 1787827160 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827150.033565:0:2396:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f7f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2396:1874672572435584:192.168.203.129@tcp:400: 00000100:00100000:2.0:1787827150.033579:0:2393:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0001-osc-ffff8cb98543f000: committing for last_committed 0 gen 1 00000100:00080000:2.0:1787827150.033594:0:2393:0:(import.c:1132:ptlrpc_connect_interpret()) lustre-OST0001-osc-ffff8cb98543f000: connect to target with instance 1 00000100:00080000:2.0:1787827150.033603:0:2393:0:(import.c:956:ptlrpc_connect_set_flags()) lustre-OST0001-osc-ffff8cb98543f000: Resetting ns_connect_flags to server flags: 0xa0427af2e3440038 00000100:00080000:2.0:1787827150.033613:0:2393:0:(import.c:1213:ptlrpc_connect_interpret()) connected to replayable target: lustre-OST0001_UUID 00000100:00080000:2.0:1787827150.033616:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab7800 lustre-OST0001_UUID: changing import state from CONNECTING to FULL 00020000:01000000:2.0:1787827150.033622:0:2393:0:(lov_obd.c:398:lov_set_osc_active()) Marking OSC lustre-OST0001_UUID active 00000080:00000004:2.0:1787827150.033626:0:2393:0:(lcommon_misc.c:94:cl_ocd_update()) Changing connect_flags: 0xa0467af2e3640038 -> 0xa0427af2e3440038 00000080:00080000:2.0:1787827150.033631:0:2393:0:(lcommon_misc.c:56:cl_init_ea_size()) updating def/max_easize: 72/65536 00000100:00100000:2.0:1787827150.033641:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb992317480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572435328:192.168.203.129@tcp:8: 00010000:00080000:2.0:1787827150.033658:0:2393:0:(ldlm_request.c:1671:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=46162) req@ffff8cb992314780 x1874672572435200/t0(0) o8->lustre-OST0000-osc-ffff8cb98543f000@192.168.203.129@tcp:28/4 lens 520/416 e 0 to 0 dl 1787827160 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787827150.033667:0:2393:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8cb98543f000: committing for last_committed 0 gen 1 00000100:00080000:2.0:1787827150.033671:0:2393:0:(import.c:1132:ptlrpc_connect_interpret()) lustre-OST0000-osc-ffff8cb98543f000: connect to target with instance 1 00000100:00080000:2.0:1787827150.033675:0:2393:0:(import.c:956:ptlrpc_connect_set_flags()) lustre-OST0000-osc-ffff8cb98543f000: Resetting ns_connect_flags to server flags: 0xa0427af2e3440038 00000100:00080000:2.0:1787827150.033679:0:2393:0:(import.c:1213:ptlrpc_connect_interpret()) connected to replayable target: lustre-OST0000_UUID 00000100:00080000:2.0:1787827150.033681:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to FULL 00020000:01000000:2.0:1787827150.033685:0:2393:0:(lov_obd.c:398:lov_set_osc_active()) Marking OSC lustre-OST0000_UUID active 00000080:00000004:2.0:1787827150.033687:0:2393:0:(lcommon_misc.c:94:cl_ocd_update()) Changing connect_flags: 0xa0427af2e3440038 -> 0xa0427af2e3440038 00000080:00080000:2.0:1787827150.033690:0:2393:0:(lcommon_misc.c:56:cl_init_ea_size()) updating def/max_easize: 72/65536 00000100:00100000:2.0:1787827150.033694:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb992314780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572435200:192.168.203.129@tcp:8: 00010000:00080000:3.0:1787827150.053704:0:6141:0:(ldlm_request.c:1671:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=46162) req@ffff8cb992314b40 x1874672572435712/t0(0) o33->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:23/10 lens 448/464 e 0 to 0 dl 1787827166 ref 2 fl Rpc:RQU/600/0 rc 0/0 job:'' uid:0 gid:0 projid:0 00000100:00100000:3.0:1787827150.053724:0:6141:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb992314b40 pname:cluuid:pid:xid:nid:opc:job mount.lustre:8cef8de3-2326-42de-9d41-f4cb057ed6e8:6141:1874672572435712:192.168.203.129@tcp:33: 00000080:00200000:3.0:1787827150.053910:0:6141:0:(llite_lib.c:3356:ll_read_inode2()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8cb998d13a88) 00000080:00200000:3.0:1787827150.053928:0:6141:0:(llite_lib.c:3119:ll_update_inode()) inode=[0x200000007:0x1:0x0], updating i_size 4096 00000080:02000400:3.0:1787827150.054005:0:6141:0:(llite_lib.c:1653:ll_fill_super()) Mounted lustre-client - version 2.17.57_65_g89f8e4a 00000080:00000004:3.0:1787827150.056453:0:6141:0:(super25.c:182:lustre_fill_super()) 192.168.203.129@tcp:/lustre: Mount complete 00000080:00200008:2.0:1787827150.123781:0:6180:0:(file.c:6313:ll_getattr_dentry()) START file /:[0x200000007:0x1:0x0](ffff8cb998d13a88), request_mask 2047, flags 0, foreign 0 00000080:00200000:2.0:1787827150.123792:0:6180:0:(file.c:6208:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8cb998d13a88),name=/ 00000002:00010000:2.0:1787827150.123814:0:6180:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1787827150.123864:0:6180:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8cb98543f000 lock: ffff8cb983df0e00/0x218c3ba13469684 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 6180 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1787827150.123874:0:6180:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8cb98543f000 lock: ffff8cb983df0e00/0x218c3ba13469684 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 6180 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1787827150.123890:0:6180:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8cb98543f000 lock: ffff8cb983df0e00/0x218c3ba13469684 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 6180 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1787827150.123913:0:6180:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f7f480 pname:cluuid:pid:xid:nid:opc:job mkdir:8cef8de3-2326-42de-9d41-f4cb057ed6e8:6180:1874672572435840:192.168.203.129@tcp:101: 00000100:00100000:2.0:1787827150.123985:0:6180:0:(client.c:2715:ptlrpc_set_wait()) set ffff8cb988d76180 going to sleep for 11 seconds 00000100:00100000:2.0:1787827150.126161:0:6180:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f7f480 pname:cluuid:pid:xid:nid:opc:job mkdir:8cef8de3-2326-42de-9d41-f4cb057ed6e8:6180:1874672572435840:192.168.203.129@tcp:101: 00010000:00010000:2.0:1787827150.126180:0:6180:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8cb98543f000 lock: ffff8cb983df0e00/0x218c3ba13469684 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xb2e8bd65d71dda2f expref: -99 pid: 6180 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1787827150.126191:0:6180:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8cb98543f000 lock: ffff8cb983df0e00/0x218c3ba13469684 lrc: 4/1,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xb2e8bd65d71dda2f expref: -99 pid: 6180 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1787827150.126202:0:6180:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8cb98543f000 lock: ffff8cb983df0e00/0x218c3ba13469684 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xb2e8bd65d71dda2f expref: -99 pid: 6180 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1787827150.126217:0:6180:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8cb98543f000 lock: ffff8cb983df0e00/0x218c3ba13469684 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xb2e8bd65d71dda2f expref: -99 pid: 6180 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1787827150.126228:0:6180:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8cb98543f000 lock: ffff8cb983df0e00/0x218c3ba13469684 lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xb2e8bd65d71dda2f expref: -99 pid: 6180 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1787827150.126236:0:6180:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8cb98543f000 lock: ffff8cb983df0e00/0x218c3ba13469684 lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xb2e8bd65d71dda2f expref: -99 pid: 6180 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1787827150.126242:0:6180:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8cb98543f000 lock: ffff8cb983df0e00/0x218c3ba13469684 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xb2e8bd65d71dda2f expref: -99 pid: 6180 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:2.0:1787827150.126296:0:6180:0:(mdc_locks.c:835:mdc_finish_enqueue()) @@@ op=0x8 disposition=0xb, status=0 req@ffff8cb999f7f480 x1874672572435840/t0(0) o101->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 576/688 e 0 to 0 dl 1787827166 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'' uid:0 gid:0 projid:0 00000002:00010000:2.0:1787827150.126310:0:6180:0:(mdc_locks.c:1359:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8cb98543f000 lock: ffff8cb983df0e00/0x218c3ba13469684 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xb2e8bd65d71dda2f expref: -99 pid: 6180 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1787827150.126322:0:6180:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8cb988fee000 type 13 mode 4 res 8589934599/1 (0 0) 00000080:00200000:2.0:1787827150.126328:0:6180:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 0 00000080:00200000:2.0:1787827150.126353:0:6180:0:(llite_lib.c:3119:ll_update_inode()) inode=[0x200000007:0x1:0x0], updating i_size 4096 00000080:00010000:2.0:1787827150.126358:0:6180:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8cb998d13a88) for lock 0x218c3ba13469684 00000080:00010000:2.0:1787827150.126363:0:6180:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8cb998d13a88) 00000080:00010000:2.0:1787827150.126366:0:6180:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x218c3ba13469684 from it ffffb8e3c4483cb8 00010000:00010000:2.0:1787827150.126370:0:6180:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8cb98543f000 lock: ffff8cb983df0e00/0x218c3ba13469684 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xb2e8bd65d71dda2f expref: -99 pid: 6180 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1787827150.126377:0:6180:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8cb98543f000 lock: ffff8cb983df0e00/0x218c3ba13469684 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xb2e8bd65d71dda2f expref: -99 pid: 6180 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1787827151.042582:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200008:1.0:1787827151.557912:0:6232:0:(file.c:6313:ll_getattr_dentry()) START file /:[0x200000007:0x1:0x0](ffff8cb998d13a88), request_mask 2047, flags 0, foreign 0 00000080:00200000:1.0:1787827151.557925:0:6232:0:(file.c:6208:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8cb998d13a88),name=/ 00000002:00010000:1.0:1787827151.557950:0:6232:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1787827151.557966:0:6232:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8cb98543f000 lock: ffff8cb983df0e00/0x218c3ba13469684 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xb2e8bd65d71dda2f expref: -99 pid: 6180 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1787827151.557979:0:6232:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8cb98543f000 lock: ffff8cb983df0e00/0x218c3ba13469684 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xb2e8bd65d71dda2f expref: -99 pid: 6180 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200000:1.0:1787827151.557987:0:6232:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1787827151.557994:0:6232:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8cb998d13a88) 00000080:00010000:1.0:1787827151.557997:0:6232:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8cb998d13a88) for lock 0x218c3ba13469684 00000080:00010000:1.0:1787827151.558005:0:6232:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x218c3ba13469684 from it ffffb8e3c4773cb8 00010000:00010000:1.0:1787827151.558009:0:6232:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8cb98543f000 lock: ffff8cb983df0e00/0x218c3ba13469684 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xb2e8bd65d71dda2f expref: -99 pid: 6180 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1787827151.558029:0:6232:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8cb98543f000 lock: ffff8cb983df0e00/0x218c3ba13469684 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xb2e8bd65d71dda2f expref: -99 pid: 6180 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 1 00000080:00200008:1.0:1787827151.558512:0:6232:0:(file.c:6313:ll_getattr_dentry()) START file /:[0x200000007:0x1:0x0](ffff8cb998d13a88), request_mask 0, flags 0, foreign 0 00000080:00200000:1.0:1787827151.558516:0:6232:0:(file.c:6208:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8cb998d13a88),name=/ 00000002:00010000:1.0:1787827151.558522:0:6232:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1787827151.558528:0:6232:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8cb98543f000 lock: ffff8cb983df0e00/0x218c3ba13469684 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xb2e8bd65d71dda2f expref: -99 pid: 6180 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 1 00010000:00010000:1.0:1787827151.558534:0:6232:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8cb98543f000 lock: ffff8cb983df0e00/0x218c3ba13469684 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xb2e8bd65d71dda2f expref: -99 pid: 6180 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 1 00000080:00200000:1.0:1787827151.558539:0:6232:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1787827151.558543:0:6232:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8cb998d13a88) 00000080:00010000:1.0:1787827151.558546:0:6232:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8cb998d13a88) for lock 0x218c3ba13469684 00000080:00010000:1.0:1787827151.558550:0:6232:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x218c3ba13469684 from it ffffb8e3c4773cb8 00010000:00010000:1.0:1787827151.558553:0:6232:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8cb98543f000 lock: ffff8cb983df0e00/0x218c3ba13469684 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xb2e8bd65d71dda2f expref: -99 pid: 6180 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 1 00010000:00010000:1.0:1787827151.558560:0:6232:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8cb98543f000 lock: ffff8cb983df0e00/0x218c3ba13469684 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xb2e8bd65d71dda2f expref: -99 pid: 6180 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 1 00000080:00200000:1.0:1787827151.558652:0:6232:0:(file.c:6208:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8cb998d13a88),name=/ 00000002:00010000:1.0:1787827151.558659:0:6232:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1787827151.558664:0:6232:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8cb98543f000 lock: ffff8cb983df0e00/0x218c3ba13469684 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xb2e8bd65d71dda2f expref: -99 pid: 6180 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 1 00010000:00010000:1.0:1787827151.558679:0:6232:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8cb98543f000 lock: ffff8cb983df0e00/0x218c3ba13469684 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xb2e8bd65d71dda2f expref: -99 pid: 6180 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 1 00000080:00200000:1.0:1787827151.558685:0:6232:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1787827151.558690:0:6232:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8cb998d13a88) 00000080:00010000:1.0:1787827151.558692:0:6232:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8cb998d13a88) for lock 0x218c3ba13469684 00000080:00010000:1.0:1787827151.558696:0:6232:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x218c3ba13469684 from it ffffb8e3c4773bb0 00010000:00010000:1.0:1787827151.558698:0:6232:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8cb98543f000 lock: ffff8cb983df0e00/0x218c3ba13469684 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xb2e8bd65d71dda2f expref: -99 pid: 6180 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 1 00010000:00010000:1.0:1787827151.558706:0:6232:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8cb98543f000 lock: ffff8cb983df0e00/0x218c3ba13469684 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xb2e8bd65d71dda2f expref: -99 pid: 6180 timeout: 0 lvb_type: 0 lru_score: 4 lru_type: 1 00000080:00200000:1.0:1787827151.558713:0:6232:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8cb998d13a88), inode mode 41ed mask 44 00000080:00200008:1.0:1787827151.558719:0:6232:0:(file.c:1090:ll_file_open()) START file /:[0x200000007:0x1:0x0](ffff8cb998d13a88), flags 100000 00000080:00200080:1.0:1787827151.558804:0:6232:0:(dir.c:2003:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8cb998d13a88) cmd=c00866a4 arg=7ffcdea53670 00800000:00000080:1.0:1787827151.558817:0:6232:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffff8cb98543f000: cmd=c00866a4 len=592 karg=ffff8cb990d5a800 uarg=0000000000000000 00800000:00000004:1.0:1787827151.558822:0:6232:0:(obd_class.h:1090:obd_statfs()) lustre-MDT0000-mdc-ffff8cb98543f000: age 107, max_age 107 00800000:00000004:1.0:1787827151.558825:0:6232:0:(obd_class.h:1130:obd_statfs()) lustre-MDT0000-mdc-ffff8cb98543f000: use ffff8cb998a3d178 cache blocks 317411/353529 objects 1023725/1024000 00000080:00200080:1.0:1787827151.558842:0:6232:0:(dir.c:2003:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8cb998d13a88) cmd=c00866a4 arg=7ffcdea53670 00800000:00000080:1.0:1787827151.558847:0:6232:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffff8cb98543f000: cmd=c00866a4 len=592 karg=ffff8cb990d5a800 uarg=0000000000000000 00800000:00000004:1.0:1787827151.558851:0:6232:0:(obd_class.h:1090:obd_statfs()) lustre-MDT0001-mdc-ffff8cb98543f000: age -894, max_age 107 00000100:00100000:1.0:1787827151.558888:0:6232:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f39e00 pname:cluuid:pid:xid:nid:opc:job lfs:8cef8de3-2326-42de-9d41-f4cb057ed6e8:6232:1874672572435968:192.168.203.129@tcp:41: 00000100:00100000:1.0:1787827151.558982:0:6232:0:(client.c:2715:ptlrpc_set_wait()) set ffff8cb988e2c540 going to sleep for 11 seconds 00010000:00080000:1.0:1787827151.560374:0:6232:0:(ldlm_request.c:1671:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=46162) req@ffff8cb999f39e00 x1874672572435968/t0(0) o41->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 224/368 e 0 to 0 dl 1787827167 ref 2 fl Rpc:RQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787827151.560398:0:6232:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f39e00 pname:cluuid:pid:xid:nid:opc:job lfs:8cef8de3-2326-42de-9d41-f4cb057ed6e8:6232:1874672572435968:192.168.203.129@tcp:41: 00800000:00000004:1.0:1787827151.560419:0:6232:0:(obd_class.h:1116:obd_statfs()) lustre-MDT0001-mdc-ffff8cb98543f000: update ffff8cb9881c5178 cache blocks 317443/353529 objects 1023743/1024000 00000080:00200080:1.0:1787827151.560470:0:6232:0:(dir.c:2003:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8cb998d13a88) cmd=c00866a4 arg=7ffcdea53670 00800000:00000080:1.0:1787827151.560482:0:6232:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffff8cb98543f000: cmd=c00866a4 len=592 karg=ffff8cb990d5a800 uarg=0000000000000000 00000080:00200080:1.0:1787827151.560490:0:6232:0:(dir.c:2003:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8cb998d13a88) cmd=c00866a4 arg=7ffcdea53670 00800000:00000080:1.0:1787827151.560494:0:6232:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffff8cb98543f000: cmd=c00866a4 len=592 karg=ffff8cb990d5a800 uarg=0000000000000000 00000080:00200080:1.0:1787827151.560499:0:6232:0:(dir.c:2003:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8cb998d13a88) cmd=c00866a4 arg=7ffcdea53670 00800000:00000080:1.0:1787827151.560504:0:6232:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffff8cb98543f000: cmd=c00866a4 len=592 karg=ffff8cb990d5a800 uarg=0000000000000000 00000080:00200080:1.0:1787827151.560508:0:6232:0:(dir.c:2003:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8cb998d13a88) cmd=c00866a4 arg=7ffcdea53670 00800000:00000080:1.0:1787827151.560513:0:6232:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffff8cb98543f000: cmd=c00866a4 len=592 karg=ffff8cb990d5a800 uarg=0000000000000000 00000080:00200080:1.0:1787827151.560517:0:6232:0:(dir.c:2003:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8cb998d13a88) cmd=c00866a4 arg=7ffcdea53670 00800000:00000080:1.0:1787827151.560522:0:6232:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffff8cb98543f000: cmd=c00866a4 len=592 karg=ffff8cb990d5a800 uarg=0000000000000000 00000080:00200080:1.0:1787827151.560527:0:6232:0:(dir.c:2003:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8cb998d13a88) cmd=c00866a4 arg=7ffcdea53670 00800000:00000080:1.0:1787827151.560532:0:6232:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffff8cb98543f000: cmd=c00866a4 len=592 karg=ffff8cb990d5a800 uarg=0000000000000000 00000080:00200080:1.0:1787827151.560537:0:6232:0:(dir.c:2003:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8cb998d13a88) cmd=c00866a4 arg=7ffcdea53670 00800000:00000080:1.0:1787827151.560542:0:6232:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffff8cb98543f000: cmd=c00866a4 len=592 karg=ffff8cb990d5a800 uarg=0000000000000000 00000080:00200080:1.0:1787827151.560546:0:6232:0:(dir.c:2003:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8cb998d13a88) cmd=c00866a4 arg=7ffcdea53670 00800000:00000080:1.0:1787827151.560550:0:6232:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffff8cb98543f000: cmd=c00866a4 len=592 karg=ffff8cb990d5a800 uarg=0000000000000000 00000080:00200080:1.0:1787827151.560555:0:6232:0:(dir.c:2003:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8cb998d13a88) cmd=c00866a4 arg=7ffcdea53670 00800000:00000080:1.0:1787827151.560564:0:6232:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffff8cb98543f000: cmd=c00866a4 len=592 karg=ffff8cb990d5a800 uarg=0000000000000000 00000080:00200080:1.0:1787827151.560571:0:6232:0:(dir.c:2003:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8cb998d13a88) cmd=c00866a4 arg=7ffcdea53670 00800000:00000080:1.0:1787827151.560576:0:6232:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffff8cb98543f000: cmd=c00866a4 len=592 karg=ffff8cb990d5a800 uarg=0000000000000000 00000080:00200080:1.0:1787827151.560581:0:6232:0:(dir.c:2003:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8cb998d13a88) cmd=c00866a4 arg=7ffcdea53670 00800000:00000080:1.0:1787827151.560592:0:6232:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffff8cb98543f000: cmd=c00866a4 len=592 karg=ffff8cb990d5a800 uarg=0000000000000000 00000080:00200080:1.0:1787827151.560599:0:6232:0:(dir.c:2003:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8cb998d13a88) cmd=c00866a4 arg=7ffcdea53670 00800000:00000080:1.0:1787827151.560604:0:6232:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffff8cb98543f000: cmd=c00866a4 len=592 karg=ffff8cb990d5a800 uarg=0000000000000000 00000080:00200080:1.0:1787827151.560609:0:6232:0:(dir.c:2003:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8cb998d13a88) cmd=c00866a4 arg=7ffcdea53670 00800000:00000080:1.0:1787827151.560613:0:6232:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffff8cb98543f000: cmd=c00866a4 len=592 karg=ffff8cb990d5a800 uarg=0000000000000000 00000080:00200080:1.0:1787827151.560617:0:6232:0:(dir.c:2003:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8cb998d13a88) cmd=c00866a4 arg=7ffcdea53670 00800000:00000080:1.0:1787827151.560622:0:6232:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffff8cb98543f000: cmd=c00866a4 len=592 karg=ffff8cb990d5a800 uarg=0000000000000000 00000080:00200080:1.0:1787827151.560627:0:6232:0:(dir.c:2003:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8cb998d13a88) cmd=c00866a4 arg=7ffcdea53670 00800000:00000080:1.0:1787827151.560631:0:6232:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffff8cb98543f000: cmd=c00866a4 len=592 karg=ffff8cb990d5a800 uarg=0000000000000000 00000080:00200080:1.0:1787827151.560636:0:6232:0:(dir.c:2003:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8cb998d13a88) cmd=c00866a4 arg=7ffcdea53670 00800000:00000080:1.0:1787827151.560639:0:6232:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffff8cb98543f000: cmd=c00866a4 len=592 karg=ffff8cb990d5a800 uarg=0000000000000000 00000080:00200080:1.0:1787827151.560643:0:6232:0:(dir.c:2003:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8cb998d13a88) cmd=c00866a4 arg=7ffcdea53670 00800000:00000080:1.0:1787827151.560647:0:6232:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffff8cb98543f000: cmd=c00866a4 len=592 karg=ffff8cb990d5a800 uarg=0000000000000000 00000080:00200080:1.0:1787827151.560652:0:6232:0:(dir.c:2003:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8cb998d13a88) cmd=c00866a4 arg=7ffcdea53670 00800000:00000080:1.0:1787827151.560656:0:6232:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffff8cb98543f000: cmd=c00866a4 len=592 karg=ffff8cb990d5a800 uarg=0000000000000000 00000080:00200080:1.0:1787827151.560661:0:6232:0:(dir.c:2003:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8cb998d13a88) cmd=c00866a4 arg=7ffcdea53670 00800000:00000080:1.0:1787827151.560664:0:6232:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffff8cb98543f000: cmd=c00866a4 len=592 karg=ffff8cb990d5a800 uarg=0000000000000000 00000080:00200080:1.0:1787827151.560668:0:6232:0:(dir.c:2003:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8cb998d13a88) cmd=c00866a4 arg=7ffcdea53670 00800000:00000080:1.0:1787827151.560672:0:6232:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffff8cb98543f000: cmd=c00866a4 len=592 karg=ffff8cb990d5a800 uarg=0000000000000000 00000080:00200080:1.0:1787827151.560676:0:6232:0:(dir.c:2003:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8cb998d13a88) cmd=c00866a4 arg=7ffcdea53670 00800000:00000080:1.0:1787827151.560680:0:6232:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffff8cb98543f000: cmd=c00866a4 len=592 karg=ffff8cb990d5a800 uarg=0000000000000000 00000080:00200080:1.0:1787827151.560685:0:6232:0:(dir.c:2003:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8cb998d13a88) cmd=c00866a4 arg=7ffcdea53670 00800000:00000080:1.0:1787827151.560689:0:6232:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffff8cb98543f000: cmd=c00866a4 len=592 karg=ffff8cb990d5a800 uarg=0000000000000000 00000080:00200080:1.0:1787827151.560693:0:6232:0:(dir.c:2003:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8cb998d13a88) cmd=c00866a4 arg=7ffcdea53670 00800000:00000080:1.0:1787827151.560697:0:6232:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffff8cb98543f000: cmd=c00866a4 len=592 karg=ffff8cb990d5a800 uarg=0000000000000000 00000080:00200080:1.0:1787827151.560707:0:6232:0:(dir.c:2003:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8cb998d13a88) cmd=c00866a4 arg=7ffcdea53670 00800000:00000080:1.0:1787827151.560712:0:6232:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffff8cb98543f000: cmd=c00866a4 len=592 karg=ffff8cb990d5a800 uarg=0000000000000000 00000080:00200080:1.0:1787827151.560716:0:6232:0:(dir.c:2003:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8cb998d13a88) cmd=c00866a4 arg=7ffcdea53670 00800000:00000080:1.0:1787827151.560719:0:6232:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffff8cb98543f000: cmd=c00866a4 len=592 karg=ffff8cb990d5a800 uarg=0000000000000000 00000080:00200080:1.0:1787827151.560723:0:6232:0:(dir.c:2003:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8cb998d13a88) cmd=c00866a4 arg=7ffcdea53670 00800000:00000080:1.0:1787827151.560727:0:6232:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffff8cb98543f000: cmd=c00866a4 len=592 karg=ffff8cb990d5a800 uarg=0000000000000000 00000080:00200080:1.0:1787827151.560731:0:6232:0:(dir.c:2003:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8cb998d13a88) cmd=c00866a4 arg=7ffcdea53670 00800000:00000080:1.0:1787827151.560734:0:6232:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffff8cb98543f000: cmd=c00866a4 len=592 karg=ffff8cb990d5a800 uarg=0000000000000000 00000080:00200080:1.0:1787827151.560738:0:6232:0:(dir.c:2003:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8cb998d13a88) cmd=c00866a4 arg=7ffcdea53670 00800000:00000080:1.0:1787827151.560741:0:6232:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffff8cb98543f000: cmd=c00866a4 len=592 karg=ffff8cb990d5a800 uarg=0000000000000000 00000080:00200080:1.0:1787827151.560746:0:6232:0:(dir.c:2003:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8cb998d13a88) cmd=c00866a4 arg=7ffcdea53670 00800000:00000080:1.0:1787827151.560749:0:6232:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffff8cb98543f000: cmd=c00866a4 len=592 karg=ffff8cb990d5a800 uarg=0000000000000000 00000080:00200080:1.0:1787827151.560753:0:6232:0:(dir.c:2003:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8cb998d13a88) cmd=c00866a4 arg=7ffcdea53670 00800000:00000080:1.0:1787827151.560757:0:6232:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffff8cb98543f000: cmd=c00866a4 len=592 karg=ffff8cb990d5a800 uarg=0000000000000000 00000080:00200080:1.0:1787827151.560761:0:6232:0:(dir.c:2003:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8cb998d13a88) cmd=c00866a4 arg=7ffcdea53670 00800000:00000080:1.0:1787827151.560765:0:6232:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffff8cb98543f000: cmd=c00866a4 len=592 karg=ffff8cb990d5a800 uarg=0000000000000000 00000080:00200080:1.0:1787827151.560769:0:6232:0:(dir.c:2003:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8cb998d13a88) cmd=c00866a4 arg=7ffcdea53670 00800000:00000080:1.0:1787827151.560773:0:6232:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffff8cb98543f000: cmd=c00866a4 len=592 karg=ffff8cb990d5a800 uarg=0000000000000000 00000080:00200080:1.0:1787827151.560777:0:6232:0:(dir.c:2003:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8cb998d13a88) cmd=c00866a4 arg=7ffcdea53670 00800000:00000080:1.0:1787827151.560780:0:6232:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffff8cb98543f000: cmd=c00866a4 len=592 karg=ffff8cb990d5a800 uarg=0000000000000000 00000080:00200080:1.0:1787827151.560784:0:6232:0:(dir.c:2003:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8cb998d13a88) cmd=c00866a4 arg=7ffcdea53670 00800000:00000080:1.0:1787827151.560787:0:6232:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffff8cb98543f000: cmd=c00866a4 len=592 karg=ffff8cb990d5a800 uarg=0000000000000000 00000080:00200080:1.0:1787827151.560800:0:6232:0:(dir.c:2003:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8cb998d13a88) cmd=c00866a4 arg=7ffcdea53670 00800000:00000080:1.0:1787827151.560804:0:6232:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffff8cb98543f000: cmd=c00866a4 len=592 karg=ffff8cb990d5a800 uarg=0000000000000000 00000080:00200080:1.0:1787827151.560812:0:6232:0:(dir.c:2003:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8cb998d13a88) cmd=c00866a4 arg=7ffcdea53670 00800000:00000080:1.0:1787827151.560821:0:6232:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffff8cb98543f000: cmd=c00866a4 len=592 karg=ffff8cb990d5a800 uarg=0000000000000000 00000080:00200080:1.0:1787827151.560827:0:6232:0:(dir.c:2003:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8cb998d13a88) cmd=c00866a4 arg=7ffcdea53670 00800000:00000080:1.0:1787827151.560831:0:6232:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffff8cb98543f000: cmd=c00866a4 len=592 karg=ffff8cb990d5a800 uarg=0000000000000000 00000080:00200080:1.0:1787827151.560835:0:6232:0:(dir.c:2003:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8cb998d13a88) cmd=c00866a4 arg=7ffcdea53670 00800000:00000080:1.0:1787827151.560838:0:6232:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffff8cb98543f000: cmd=c00866a4 len=592 karg=ffff8cb990d5a800 uarg=0000000000000000 00000080:00200080:1.0:1787827151.560842:0:6232:0:(dir.c:2003:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8cb998d13a88) cmd=c00866a4 arg=7ffcdea53670 00800000:00000080:1.0:1787827151.560845:0:6232:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffff8cb98543f000: cmd=c00866a4 len=592 karg=ffff8cb990d5a800 uarg=0000000000000000 00000080:00200080:1.0:1787827151.560849:0:6232:0:(dir.c:2003:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8cb998d13a88) cmd=c00866a4 arg=7ffcdea53670 00800000:00000080:1.0:1787827151.560853:0:6232:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffff8cb98543f000: cmd=c00866a4 len=592 karg=ffff8cb990d5a800 uarg=0000000000000000 00000080:00200080:1.0:1787827151.560857:0:6232:0:(dir.c:2003:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8cb998d13a88) cmd=c00866a4 arg=7ffcdea53670 00800000:00000080:1.0:1787827151.560860:0:6232:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffff8cb98543f000: cmd=c00866a4 len=592 karg=ffff8cb990d5a800 uarg=0000000000000000 00000080:00200080:1.0:1787827151.560864:0:6232:0:(dir.c:2003:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8cb998d13a88) cmd=c00866a4 arg=7ffcdea53670 00800000:00000080:1.0:1787827151.560868:0:6232:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffff8cb98543f000: cmd=c00866a4 len=592 karg=ffff8cb990d5a800 uarg=0000000000000000 00000080:00200080:1.0:1787827151.560872:0:6232:0:(dir.c:2003:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8cb998d13a88) cmd=c00866a4 arg=7ffcdea53670 00800000:00000080:1.0:1787827151.560875:0:6232:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffff8cb98543f000: cmd=c00866a4 len=592 karg=ffff8cb990d5a800 uarg=0000000000000000 00000080:00200080:1.0:1787827151.560879:0:6232:0:(dir.c:2003:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8cb998d13a88) cmd=c00866a4 arg=7ffcdea53670 00800000:00000080:1.0:1787827151.560882:0:6232:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffff8cb98543f000: cmd=c00866a4 len=592 karg=ffff8cb990d5a800 uarg=0000000000000000 00000080:00200080:1.0:1787827151.560887:0:6232:0:(dir.c:2003:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8cb998d13a88) cmd=c00866a4 arg=7ffcdea53670 00800000:00000080:1.0:1787827151.560890:0:6232:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffff8cb98543f000: cmd=c00866a4 len=592 karg=ffff8cb990d5a800 uarg=0000000000000000 00000080:00200080:1.0:1787827151.560894:0:6232:0:(dir.c:2003:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8cb998d13a88) cmd=c00866a4 arg=7ffcdea53670 00800000:00000080:1.0:1787827151.560897:0:6232:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffff8cb98543f000: cmd=c00866a4 len=592 karg=ffff8cb990d5a800 uarg=0000000000000000 00000080:00200080:1.0:1787827151.560901:0:6232:0:(dir.c:2003:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8cb998d13a88) cmd=c00866a4 arg=7ffcdea53670 00800000:00000080:1.0:1787827151.560904:0:6232:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffff8cb98543f000: cmd=c00866a4 len=592 karg=ffff8cb990d5a800 uarg=0000000000000000 00000080:00200080:1.0:1787827151.560908:0:6232:0:(dir.c:2003:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8cb998d13a88) cmd=c00866a4 arg=7ffcdea53670 00800000:00000080:1.0:1787827151.560911:0:6232:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffff8cb98543f000: cmd=c00866a4 len=592 karg=ffff8cb990d5a800 uarg=0000000000000000 00000080:00200080:1.0:1787827151.560921:0:6232:0:(dir.c:2003:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8cb998d13a88) cmd=c00866a4 arg=7ffcdea53670 00800000:00000080:1.0:1787827151.560925:0:6232:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffff8cb98543f000: cmd=c00866a4 len=592 karg=ffff8cb990d5a800 uarg=0000000000000000 00000080:00200080:1.0:1787827151.560929:0:6232:0:(dir.c:2003:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8cb998d13a88) cmd=c00866a4 arg=7ffcdea53670 00800000:00000080:1.0:1787827151.560932:0:6232:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffff8cb98543f000: cmd=c00866a4 len=592 karg=ffff8cb990d5a800 uarg=0000000000000000 00000080:00200080:1.0:1787827151.560937:0:6232:0:(dir.c:2003:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8cb998d13a88) cmd=c00866a4 arg=7ffcdea53670 00800000:00000080:1.0:1787827151.560940:0:6232:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffff8cb98543f000: cmd=c00866a4 len=592 karg=ffff8cb990d5a800 uarg=0000000000000000 00000080:00200080:1.0:1787827151.560945:0:6232:0:(dir.c:2003:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8cb998d13a88) cmd=c00866a4 arg=7ffcdea53670 00800000:00000080:1.0:1787827151.560948:0:6232:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffff8cb98543f000: cmd=c00866a4 len=592 karg=ffff8cb990d5a800 uarg=0000000000000000 00000080:00200080:1.0:1787827151.560952:0:6232:0:(dir.c:2003:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8cb998d13a88) cmd=c00866a4 arg=7ffcdea53670 00800000:00000080:1.0:1787827151.560956:0:6232:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffff8cb98543f000: cmd=c00866a4 len=592 karg=ffff8cb990d5a800 uarg=0000000000000000 00000080:00200080:1.0:1787827151.560960:0:6232:0:(dir.c:2003:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8cb998d13a88) cmd=c00866a4 arg=7ffcdea53670 00800000:00000080:1.0:1787827151.560963:0:6232:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffff8cb98543f000: cmd=c00866a4 len=592 karg=ffff8cb990d5a800 uarg=0000000000000000 00000080:00200080:1.0:1787827151.560967:0:6232:0:(dir.c:2003:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8cb998d13a88) cmd=c00866a4 arg=7ffcdea53670 00800000:00000080:1.0:1787827151.560971:0:6232:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffff8cb98543f000: cmd=c00866a4 len=592 karg=ffff8cb990d5a800 uarg=0000000000000000 00000080:00200080:1.0:1787827151.560975:0:6232:0:(dir.c:2003:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8cb998d13a88) cmd=c00866a4 arg=7ffcdea53670 00800000:00000080:1.0:1787827151.560979:0:6232:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffff8cb98543f000: cmd=c00866a4 len=592 karg=ffff8cb990d5a800 uarg=0000000000000000 00000080:00200080:1.0:1787827151.560983:0:6232:0:(dir.c:2003:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8cb998d13a88) cmd=c00866a4 arg=7ffcdea53670 00800000:00000080:1.0:1787827151.560986:0:6232:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffff8cb98543f000: cmd=c00866a4 len=592 karg=ffff8cb990d5a800 uarg=0000000000000000 00000080:00200080:1.0:1787827151.560990:0:6232:0:(dir.c:2003:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8cb998d13a88) cmd=c00866a4 arg=7ffcdea53670 00800000:00000080:1.0:1787827151.560993:0:6232:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffff8cb98543f000: cmd=c00866a4 len=592 karg=ffff8cb990d5a800 uarg=0000000000000000 00000080:00200080:1.0:1787827151.560998:0:6232:0:(dir.c:2003:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8cb998d13a88) cmd=c00866a4 arg=7ffcdea53670 00800000:00000080:1.0:1787827151.561001:0:6232:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffff8cb98543f000: cmd=c00866a4 len=592 karg=ffff8cb990d5a800 uarg=0000000000000000 00000080:00200080:1.0:1787827151.561005:0:6232:0:(dir.c:2003:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8cb998d13a88) cmd=c00866a4 arg=7ffcdea53670 00800000:00000080:1.0:1787827151.561009:0:6232:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffff8cb98543f000: cmd=c00866a4 len=592 karg=ffff8cb990d5a800 uarg=0000000000000000 00000080:00200080:1.0:1787827151.561012:0:6232:0:(dir.c:2003:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8cb998d13a88) cmd=c00866a4 arg=7ffcdea53670 00800000:00000080:1.0:1787827151.561021:0:6232:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffff8cb98543f000: cmd=c00866a4 len=592 karg=ffff8cb990d5a800 uarg=0000000000000000 00000080:00200080:1.0:1787827151.561026:0:6232:0:(dir.c:2003:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8cb998d13a88) cmd=c00866a4 arg=7ffcdea53670 00800000:00000080:1.0:1787827151.561030:0:6232:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffff8cb98543f000: cmd=c00866a4 len=592 karg=ffff8cb990d5a800 uarg=0000000000000000 00000080:00200080:1.0:1787827151.561034:0:6232:0:(dir.c:2003:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8cb998d13a88) cmd=c00866a4 arg=7ffcdea53670 00800000:00000080:1.0:1787827151.561038:0:6232:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffff8cb98543f000: cmd=c00866a4 len=592 karg=ffff8cb990d5a800 uarg=0000000000000000 00000080:00200080:1.0:1787827151.561042:0:6232:0:(dir.c:2003:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8cb998d13a88) cmd=c00866a4 arg=7ffcdea53670 00020000:00000080:1.0:1787827151.561048:0:6232:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffff8cb98543f000: cmd=c00866a4 len=592 karg=ffff8cb990d5a800 uarg=0000000000000000 00020000:00000004:1.0:1787827151.561054:0:6232:0:(obd_class.h:1090:obd_statfs()) lustre-OST0000-osc-ffff8cb98543f000: age -894, max_age 107 00000100:00100000:1.0:1787827151.561079:0:6232:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f39e00 pname:cluuid:pid:xid:nid:opc:job lfs:8cef8de3-2326-42de-9d41-f4cb057ed6e8:6232:1874672572436096:192.168.203.129@tcp:13: 00000100:00100000:1.0:1787827151.561136:0:6232:0:(client.c:2715:ptlrpc_set_wait()) set ffff8cb988e2c540 going to sleep for 11 seconds 00010000:00080000:1.0:1787827151.562331:0:6232:0:(ldlm_request.c:1671:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=46162) req@ffff8cb999f39e00 x1874672572436096/t0(0) o13->lustre-OST0000-osc-ffff8cb98543f000@192.168.203.129@tcp:7/4 lens 224/368 e 0 to 0 dl 1787827167 ref 2 fl Rpc:RQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787827151.562347:0:6232:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f39e00 pname:cluuid:pid:xid:nid:opc:job lfs:8cef8de3-2326-42de-9d41-f4cb057ed6e8:6232:1874672572436096:192.168.203.129@tcp:13: 00020000:00000004:1.0:1787827151.562366:0:6232:0:(obd_class.h:1116:obd_statfs()) lustre-OST0000-osc-ffff8cb98543f000: update ffff8cb9881c1e08 cache blocks 901369/958279 objects 261775/262144 00000080:00200080:1.0:1787827151.562412:0:6232:0:(dir.c:2003:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8cb998d13a88) cmd=c00866a4 arg=7ffcdea53670 00020000:00000080:1.0:1787827151.562425:0:6232:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffff8cb98543f000: cmd=c00866a4 len=592 karg=ffff8cb990d5a800 uarg=0000000000000000 00020000:00000004:1.0:1787827151.562432:0:6232:0:(obd_class.h:1090:obd_statfs()) lustre-OST0001-osc-ffff8cb98543f000: age -894, max_age 107 00000100:00100000:1.0:1787827151.562453:0:6232:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f39e00 pname:cluuid:pid:xid:nid:opc:job lfs:8cef8de3-2326-42de-9d41-f4cb057ed6e8:6232:1874672572436224:192.168.203.129@tcp:13: 00000100:00100000:1.0:1787827151.562513:0:6232:0:(client.c:2715:ptlrpc_set_wait()) set ffff8cb988e2c540 going to sleep for 11 seconds 00010000:00080000:1.0:1787827151.563342:0:6232:0:(ldlm_request.c:1671:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=46162) req@ffff8cb999f39e00 x1874672572436224/t0(0) o13->lustre-OST0001-osc-ffff8cb98543f000@192.168.203.129@tcp:7/4 lens 224/368 e 0 to 0 dl 1787827167 ref 2 fl Rpc:RQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787827151.563355:0:6232:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f39e00 pname:cluuid:pid:xid:nid:opc:job lfs:8cef8de3-2326-42de-9d41-f4cb057ed6e8:6232:1874672572436224:192.168.203.129@tcp:13: 00020000:00000004:1.0:1787827151.563369:0:6232:0:(obd_class.h:1116:obd_statfs()) lustre-OST0001-osc-ffff8cb98543f000: update ffff8cb9881c37c0 cache blocks 901369/958279 objects 261775/262144 00000080:00200080:1.0:1787827151.563402:0:6232:0:(dir.c:2003:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8cb998d13a88) cmd=c00866a4 arg=7ffcdea53670 00020000:00000080:1.0:1787827151.563424:0:6232:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffff8cb98543f000: cmd=c00866a4 len=592 karg=ffff8cb990d5a800 uarg=0000000000000000 00000080:00200080:1.0:1787827151.563432:0:6232:0:(dir.c:2003:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8cb998d13a88) cmd=c00866a4 arg=7ffcdea53670 00020000:00000080:1.0:1787827151.563438:0:6232:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffff8cb98543f000: cmd=c00866a4 len=592 karg=ffff8cb990d5a800 uarg=0000000000000000 00000080:00200080:1.0:1787827151.563443:0:6232:0:(dir.c:2003:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8cb998d13a88) cmd=c00866a4 arg=7ffcdea53670 00020000:00000080:1.0:1787827151.563447:0:6232:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffff8cb98543f000: cmd=c00866a4 len=592 karg=ffff8cb990d5a800 uarg=0000000000000000 00000080:00200080:1.0:1787827151.563451:0:6232:0:(dir.c:2003:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8cb998d13a88) cmd=c00866a4 arg=7ffcdea53670 00020000:00000080:1.0:1787827151.563469:0:6232:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffff8cb98543f000: cmd=c00866a4 len=592 karg=ffff8cb990d5a800 uarg=0000000000000000 00000080:00200080:1.0:1787827151.563475:0:6232:0:(dir.c:2003:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8cb998d13a88) cmd=c00866a4 arg=7ffcdea53670 00020000:00000080:1.0:1787827151.563480:0:6232:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffff8cb98543f000: cmd=c00866a4 len=592 karg=ffff8cb990d5a800 uarg=0000000000000000 00000080:00200080:1.0:1787827151.563485:0:6232:0:(dir.c:2003:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8cb998d13a88) cmd=c00866a4 arg=7ffcdea53670 00020000:00000080:1.0:1787827151.563490:0:6232:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffff8cb98543f000: cmd=c00866a4 len=592 karg=ffff8cb990d5a800 uarg=0000000000000000 00000080:00200080:1.0:1787827151.563494:0:6232:0:(dir.c:2003:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8cb998d13a88) cmd=c00866a4 arg=7ffcdea53670 00020000:00000080:1.0:1787827151.563498:0:6232:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffff8cb98543f000: cmd=c00866a4 len=592 karg=ffff8cb990d5a800 uarg=0000000000000000 00000080:00200080:1.0:1787827151.563502:0:6232:0:(dir.c:2003:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8cb998d13a88) cmd=c00866a4 arg=7ffcdea53670 00020000:00000080:1.0:1787827151.563507:0:6232:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffff8cb98543f000: cmd=c00866a4 len=592 karg=ffff8cb990d5a800 uarg=0000000000000000 00000080:00200080:1.0:1787827151.563512:0:6232:0:(dir.c:2003:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8cb998d13a88) cmd=c00866a4 arg=7ffcdea53670 00020000:00000080:1.0:1787827151.563515:0:6232:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffff8cb98543f000: cmd=c00866a4 len=592 karg=ffff8cb990d5a800 uarg=0000000000000000 00000080:00200080:1.0:1787827151.563521:0:6232:0:(dir.c:2003:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8cb998d13a88) cmd=c00866a4 arg=7ffcdea53670 00020000:00000080:1.0:1787827151.563526:0:6232:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffff8cb98543f000: cmd=c00866a4 len=592 karg=ffff8cb990d5a800 uarg=0000000000000000 00000080:00200080:1.0:1787827151.563531:0:6232:0:(dir.c:2003:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8cb998d13a88) cmd=c00866a4 arg=7ffcdea53670 00020000:00000080:1.0:1787827151.563535:0:6232:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffff8cb98543f000: cmd=c00866a4 len=592 karg=ffff8cb990d5a800 uarg=0000000000000000 00000080:00200080:1.0:1787827151.563540:0:6232:0:(dir.c:2003:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8cb998d13a88) cmd=c00866a4 arg=7ffcdea53670 00020000:00000080:1.0:1787827151.563544:0:6232:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffff8cb98543f000: cmd=c00866a4 len=592 karg=ffff8cb990d5a800 uarg=0000000000000000 00000080:00200080:1.0:1787827151.563550:0:6232:0:(dir.c:2003:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8cb998d13a88) cmd=c00866a4 arg=7ffcdea53670 00020000:00000080:1.0:1787827151.563554:0:6232:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffff8cb98543f000: cmd=c00866a4 len=592 karg=ffff8cb990d5a800 uarg=0000000000000000 00000080:00200080:1.0:1787827151.563566:0:6232:0:(dir.c:2003:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8cb998d13a88) cmd=c00866a4 arg=7ffcdea53670 00020000:00000080:1.0:1787827151.563572:0:6232:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffff8cb98543f000: cmd=c00866a4 len=592 karg=ffff8cb990d5a800 uarg=0000000000000000 00000080:00200080:1.0:1787827151.563577:0:6232:0:(dir.c:2003:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8cb998d13a88) cmd=c00866a4 arg=7ffcdea53670 00020000:00000080:1.0:1787827151.563582:0:6232:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffff8cb98543f000: cmd=c00866a4 len=592 karg=ffff8cb990d5a800 uarg=0000000000000000 00000080:00200080:1.0:1787827151.563587:0:6232:0:(dir.c:2003:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8cb998d13a88) cmd=c00866a4 arg=7ffcdea53670 00020000:00000080:1.0:1787827151.563591:0:6232:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffff8cb98543f000: cmd=c00866a4 len=592 karg=ffff8cb990d5a800 uarg=0000000000000000 00000080:00200080:1.0:1787827151.563596:0:6232:0:(dir.c:2003:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8cb998d13a88) cmd=c00866a4 arg=7ffcdea53670 00020000:00000080:1.0:1787827151.563601:0:6232:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffff8cb98543f000: cmd=c00866a4 len=592 karg=ffff8cb990d5a800 uarg=0000000000000000 00000080:00200080:1.0:1787827151.563606:0:6232:0:(dir.c:2003:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8cb998d13a88) cmd=c00866a4 arg=7ffcdea53670 00020000:00000080:1.0:1787827151.563610:0:6232:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffff8cb98543f000: cmd=c00866a4 len=592 karg=ffff8cb990d5a800 uarg=0000000000000000 00000080:00200080:1.0:1787827151.563614:0:6232:0:(dir.c:2003:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8cb998d13a88) cmd=c00866a4 arg=7ffcdea53670 00020000:00000080:1.0:1787827151.563618:0:6232:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffff8cb98543f000: cmd=c00866a4 len=592 karg=ffff8cb990d5a800 uarg=0000000000000000 00000080:00200080:1.0:1787827151.563623:0:6232:0:(dir.c:2003:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8cb998d13a88) cmd=c00866a4 arg=7ffcdea53670 00020000:00000080:1.0:1787827151.563627:0:6232:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffff8cb98543f000: cmd=c00866a4 len=592 karg=ffff8cb990d5a800 uarg=0000000000000000 00000080:00200080:1.0:1787827151.563632:0:6232:0:(dir.c:2003:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8cb998d13a88) cmd=c00866a4 arg=7ffcdea53670 00020000:00000080:1.0:1787827151.563636:0:6232:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffff8cb98543f000: cmd=c00866a4 len=592 karg=ffff8cb990d5a800 uarg=0000000000000000 00000080:00200080:1.0:1787827151.563640:0:6232:0:(dir.c:2003:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8cb998d13a88) cmd=c00866a4 arg=7ffcdea53670 00020000:00000080:1.0:1787827151.563644:0:6232:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffff8cb98543f000: cmd=c00866a4 len=592 karg=ffff8cb990d5a800 uarg=0000000000000000 00000080:00200080:1.0:1787827151.563649:0:6232:0:(dir.c:2003:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8cb998d13a88) cmd=c00866a4 arg=7ffcdea53670 00020000:00000080:1.0:1787827151.563654:0:6232:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffff8cb98543f000: cmd=c00866a4 len=592 karg=ffff8cb990d5a800 uarg=0000000000000000 00000080:00200080:1.0:1787827151.563658:0:6232:0:(dir.c:2003:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8cb998d13a88) cmd=c00866a4 arg=7ffcdea53670 00020000:00000080:1.0:1787827151.563662:0:6232:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffff8cb98543f000: cmd=c00866a4 len=592 karg=ffff8cb990d5a800 uarg=0000000000000000 00000080:00200080:1.0:1787827151.563667:0:6232:0:(dir.c:2003:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8cb998d13a88) cmd=c00866a4 arg=7ffcdea53670 00020000:00000080:1.0:1787827151.563680:0:6232:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffff8cb98543f000: cmd=c00866a4 len=592 karg=ffff8cb990d5a800 uarg=0000000000000000 00000080:00200080:1.0:1787827151.563687:0:6232:0:(dir.c:2003:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8cb998d13a88) cmd=c00866a4 arg=7ffcdea53670 00020000:00000080:1.0:1787827151.563698:0:6232:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffff8cb98543f000: cmd=c00866a4 len=592 karg=ffff8cb990d5a800 uarg=0000000000000000 00000080:00200080:1.0:1787827151.563704:0:6232:0:(dir.c:2003:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8cb998d13a88) cmd=c00866a4 arg=7ffcdea53670 00020000:00000080:1.0:1787827151.563708:0:6232:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffff8cb98543f000: cmd=c00866a4 len=592 karg=ffff8cb990d5a800 uarg=0000000000000000 00000080:00200080:1.0:1787827151.563712:0:6232:0:(dir.c:2003:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8cb998d13a88) cmd=c00866a4 arg=7ffcdea53670 00020000:00000080:1.0:1787827151.563716:0:6232:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffff8cb98543f000: cmd=c00866a4 len=592 karg=ffff8cb990d5a800 uarg=0000000000000000 00000080:00200080:1.0:1787827151.563721:0:6232:0:(dir.c:2003:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8cb998d13a88) cmd=c00866a4 arg=7ffcdea53670 00020000:00000080:1.0:1787827151.563725:0:6232:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffff8cb98543f000: cmd=c00866a4 len=592 karg=ffff8cb990d5a800 uarg=0000000000000000 00000080:00200080:1.0:1787827151.563729:0:6232:0:(dir.c:2003:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8cb998d13a88) cmd=c00866a4 arg=7ffcdea53670 00020000:00000080:1.0:1787827151.563733:0:6232:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffff8cb98543f000: cmd=c00866a4 len=592 karg=ffff8cb990d5a800 uarg=0000000000000000 00000080:00200080:1.0:1787827151.563738:0:6232:0:(dir.c:2003:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8cb998d13a88) cmd=c00866a4 arg=7ffcdea53670 00020000:00000080:1.0:1787827151.563741:0:6232:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffff8cb98543f000: cmd=c00866a4 len=592 karg=ffff8cb990d5a800 uarg=0000000000000000 00000080:00200080:1.0:1787827151.563745:0:6232:0:(dir.c:2003:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8cb998d13a88) cmd=c00866a4 arg=7ffcdea53670 00020000:00000080:1.0:1787827151.563749:0:6232:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffff8cb98543f000: cmd=c00866a4 len=592 karg=ffff8cb990d5a800 uarg=0000000000000000 00000080:00200080:1.0:1787827151.563753:0:6232:0:(dir.c:2003:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8cb998d13a88) cmd=c00866a4 arg=7ffcdea53670 00020000:00000080:1.0:1787827151.563756:0:6232:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffff8cb98543f000: cmd=c00866a4 len=592 karg=ffff8cb990d5a800 uarg=0000000000000000 00000080:00200080:1.0:1787827151.563761:0:6232:0:(dir.c:2003:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8cb998d13a88) cmd=c00866a4 arg=7ffcdea53670 00020000:00000080:1.0:1787827151.563764:0:6232:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffff8cb98543f000: cmd=c00866a4 len=592 karg=ffff8cb990d5a800 uarg=0000000000000000 00000080:00200080:1.0:1787827151.563768:0:6232:0:(dir.c:2003:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8cb998d13a88) cmd=c00866a4 arg=7ffcdea53670 00020000:00000080:1.0:1787827151.563772:0:6232:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffff8cb98543f000: cmd=c00866a4 len=592 karg=ffff8cb990d5a800 uarg=0000000000000000 00000080:00200080:1.0:1787827151.563776:0:6232:0:(dir.c:2003:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8cb998d13a88) cmd=c00866a4 arg=7ffcdea53670 00020000:00000080:1.0:1787827151.563780:0:6232:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffff8cb98543f000: cmd=c00866a4 len=592 karg=ffff8cb990d5a800 uarg=0000000000000000 00000080:00200080:1.0:1787827151.563784:0:6232:0:(dir.c:2003:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8cb998d13a88) cmd=c00866a4 arg=7ffcdea53670 00020000:00000080:1.0:1787827151.563788:0:6232:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffff8cb98543f000: cmd=c00866a4 len=592 karg=ffff8cb990d5a800 uarg=0000000000000000 00000080:00200080:1.0:1787827151.563800:0:6232:0:(dir.c:2003:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8cb998d13a88) cmd=c00866a4 arg=7ffcdea53670 00020000:00000080:1.0:1787827151.563803:0:6232:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffff8cb98543f000: cmd=c00866a4 len=592 karg=ffff8cb990d5a800 uarg=0000000000000000 00000080:00200080:1.0:1787827151.563814:0:6232:0:(dir.c:2003:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8cb998d13a88) cmd=c00866a4 arg=7ffcdea53670 00020000:00000080:1.0:1787827151.563818:0:6232:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffff8cb98543f000: cmd=c00866a4 len=592 karg=ffff8cb990d5a800 uarg=0000000000000000 00000080:00200080:1.0:1787827151.563822:0:6232:0:(dir.c:2003:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8cb998d13a88) cmd=c00866a4 arg=7ffcdea53670 00020000:00000080:1.0:1787827151.563825:0:6232:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffff8cb98543f000: cmd=c00866a4 len=592 karg=ffff8cb990d5a800 uarg=0000000000000000 00000080:00200080:1.0:1787827151.563829:0:6232:0:(dir.c:2003:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8cb998d13a88) cmd=c00866a4 arg=7ffcdea53670 00020000:00000080:1.0:1787827151.563833:0:6232:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffff8cb98543f000: cmd=c00866a4 len=592 karg=ffff8cb990d5a800 uarg=0000000000000000 00000080:00200080:1.0:1787827151.563837:0:6232:0:(dir.c:2003:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8cb998d13a88) cmd=c00866a4 arg=7ffcdea53670 00020000:00000080:1.0:1787827151.563840:0:6232:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffff8cb98543f000: cmd=c00866a4 len=592 karg=ffff8cb990d5a800 uarg=0000000000000000 00000080:00200080:1.0:1787827151.563844:0:6232:0:(dir.c:2003:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8cb998d13a88) cmd=c00866a4 arg=7ffcdea53670 00020000:00000080:1.0:1787827151.563847:0:6232:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffff8cb98543f000: cmd=c00866a4 len=592 karg=ffff8cb990d5a800 uarg=0000000000000000 00000080:00200080:1.0:1787827151.563851:0:6232:0:(dir.c:2003:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8cb998d13a88) cmd=c00866a4 arg=7ffcdea53670 00020000:00000080:1.0:1787827151.563854:0:6232:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffff8cb98543f000: cmd=c00866a4 len=592 karg=ffff8cb990d5a800 uarg=0000000000000000 00000080:00200080:1.0:1787827151.563858:0:6232:0:(dir.c:2003:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8cb998d13a88) cmd=c00866a4 arg=7ffcdea53670 00020000:00000080:1.0:1787827151.563861:0:6232:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffff8cb98543f000: cmd=c00866a4 len=592 karg=ffff8cb990d5a800 uarg=0000000000000000 00000080:00200080:1.0:1787827151.563866:0:6232:0:(dir.c:2003:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8cb998d13a88) cmd=c00866a4 arg=7ffcdea53670 00020000:00000080:1.0:1787827151.563870:0:6232:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffff8cb98543f000: cmd=c00866a4 len=592 karg=ffff8cb990d5a800 uarg=0000000000000000 00000080:00200080:1.0:1787827151.563874:0:6232:0:(dir.c:2003:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8cb998d13a88) cmd=c00866a4 arg=7ffcdea53670 00020000:00000080:1.0:1787827151.563878:0:6232:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffff8cb98543f000: cmd=c00866a4 len=592 karg=ffff8cb990d5a800 uarg=0000000000000000 00000080:00200080:1.0:1787827151.563882:0:6232:0:(dir.c:2003:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8cb998d13a88) cmd=c00866a4 arg=7ffcdea53670 00020000:00000080:1.0:1787827151.563886:0:6232:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffff8cb98543f000: cmd=c00866a4 len=592 karg=ffff8cb990d5a800 uarg=0000000000000000 00000080:00200080:1.0:1787827151.563890:0:6232:0:(dir.c:2003:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8cb998d13a88) cmd=c00866a4 arg=7ffcdea53670 00020000:00000080:1.0:1787827151.563899:0:6232:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffff8cb98543f000: cmd=c00866a4 len=592 karg=ffff8cb990d5a800 uarg=0000000000000000 00000080:00200080:1.0:1787827151.563904:0:6232:0:(dir.c:2003:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8cb998d13a88) cmd=c00866a4 arg=7ffcdea53670 00020000:00000080:1.0:1787827151.563908:0:6232:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffff8cb98543f000: cmd=c00866a4 len=592 karg=ffff8cb990d5a800 uarg=0000000000000000 00000080:00200080:1.0:1787827151.563912:0:6232:0:(dir.c:2003:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8cb998d13a88) cmd=c00866a4 arg=7ffcdea53670 00020000:00000080:1.0:1787827151.563921:0:6232:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffff8cb98543f000: cmd=c00866a4 len=592 karg=ffff8cb990d5a800 uarg=0000000000000000 00000080:00200080:1.0:1787827151.563926:0:6232:0:(dir.c:2003:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8cb998d13a88) cmd=c00866a4 arg=7ffcdea53670 00020000:00000080:1.0:1787827151.563930:0:6232:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffff8cb98543f000: cmd=c00866a4 len=592 karg=ffff8cb990d5a800 uarg=0000000000000000 00000080:00200080:1.0:1787827151.563934:0:6232:0:(dir.c:2003:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8cb998d13a88) cmd=c00866a4 arg=7ffcdea53670 00020000:00000080:1.0:1787827151.563938:0:6232:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffff8cb98543f000: cmd=c00866a4 len=592 karg=ffff8cb990d5a800 uarg=0000000000000000 00000080:00200080:1.0:1787827151.563942:0:6232:0:(dir.c:2003:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8cb998d13a88) cmd=c00866a4 arg=7ffcdea53670 00020000:00000080:1.0:1787827151.563946:0:6232:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffff8cb98543f000: cmd=c00866a4 len=592 karg=ffff8cb990d5a800 uarg=0000000000000000 00000080:00200080:1.0:1787827151.563950:0:6232:0:(dir.c:2003:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8cb998d13a88) cmd=c00866a4 arg=7ffcdea53670 00020000:00000080:1.0:1787827151.563954:0:6232:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffff8cb98543f000: cmd=c00866a4 len=592 karg=ffff8cb990d5a800 uarg=0000000000000000 00000080:00200080:1.0:1787827151.563958:0:6232:0:(dir.c:2003:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8cb998d13a88) cmd=c00866a4 arg=7ffcdea53670 00020000:00000080:1.0:1787827151.563961:0:6232:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffff8cb98543f000: cmd=c00866a4 len=592 karg=ffff8cb990d5a800 uarg=0000000000000000 00000080:00200080:1.0:1787827151.563965:0:6232:0:(dir.c:2003:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8cb998d13a88) cmd=c00866a4 arg=7ffcdea53670 00020000:00000080:1.0:1787827151.563968:0:6232:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffff8cb98543f000: cmd=c00866a4 len=592 karg=ffff8cb990d5a800 uarg=0000000000000000 00000080:00200080:1.0:1787827151.563972:0:6232:0:(dir.c:2003:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8cb998d13a88) cmd=c00866a4 arg=7ffcdea53670 00020000:00000080:1.0:1787827151.563975:0:6232:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffff8cb98543f000: cmd=c00866a4 len=592 karg=ffff8cb990d5a800 uarg=0000000000000000 00000080:00200080:1.0:1787827151.563988:0:6232:0:(dir.c:2003:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8cb998d13a88) cmd=c00866a4 arg=7ffcdea53670 00020000:00000080:1.0:1787827151.563992:0:6232:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffff8cb98543f000: cmd=c00866a4 len=592 karg=ffff8cb990d5a800 uarg=0000000000000000 00000080:00200080:1.0:1787827151.563997:0:6232:0:(dir.c:2003:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8cb998d13a88) cmd=c00866a4 arg=7ffcdea53670 00020000:00000080:1.0:1787827151.564000:0:6232:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffff8cb98543f000: cmd=c00866a4 len=592 karg=ffff8cb990d5a800 uarg=0000000000000000 00000080:00200080:1.0:1787827151.564005:0:6232:0:(dir.c:2003:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8cb998d13a88) cmd=c00866a4 arg=7ffcdea53670 00020000:00000080:1.0:1787827151.564008:0:6232:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffff8cb98543f000: cmd=c00866a4 len=592 karg=ffff8cb990d5a800 uarg=0000000000000000 00000080:00200080:1.0:1787827151.564012:0:6232:0:(dir.c:2003:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8cb998d13a88) cmd=c00866a4 arg=7ffcdea53670 00020000:00000080:1.0:1787827151.564016:0:6232:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffff8cb98543f000: cmd=c00866a4 len=592 karg=ffff8cb990d5a800 uarg=0000000000000000 00000080:00200080:1.0:1787827151.564020:0:6232:0:(dir.c:2003:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8cb998d13a88) cmd=c00866a4 arg=7ffcdea53670 00020000:00000080:1.0:1787827151.564023:0:6232:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffff8cb98543f000: cmd=c00866a4 len=592 karg=ffff8cb990d5a800 uarg=0000000000000000 00000080:00200008:1.0:1787827151.564042:0:6232:0:(file.c:479:ll_file_release()) START file /:[0x200000007:0x1:0x0](ffff8cb998d13a88), flags 100000 00010000:00010000:2.0:1787827152.066345:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000001:02000400:0.0:1787827152.539979:0:6425:0:(debug.c:653:libcfs_debug_mark_buffer()) DEBUG MARKER: Using TIMEOUT=20 00010000:00010000:0.0:1787827153.090383:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787827153.855843:0:2386:0:(events.c:340:request_in_callback()) peer: 12345-192.168.203.129@tcp (source: 12345-192.168.203.129@tcp) 00000100:00100000:0.0:1787827153.856111:0:2387:0:(events.c:340:request_in_callback()) peer: 12345-192.168.203.129@tcp (source: 12345-192.168.203.129@tcp) 00000100:00100000:1.0:1787827153.856498:0:6142:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1874672580865024 00000100:00100000:1.0:1787827153.856512:0:6142:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1874672580865024 00000100:00100000:1.0:1787827153.856561:0:6142:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1874672580865280 00000100:00100000:1.0:1787827153.856564:0:6142:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1874672580865280 00000100:00100000:1.0:1787827153.856583:0:6142:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.129@tcp, seq: 0 00000100:00100000:1.0:1787827153.856592:0:6142:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff8cb999c6a940 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:0bb8b105-f59c-455a-b6b1-ce40efc3999c+4:10297:x1874672580865024:12345-192.168.203.129@tcp:104: 00010000:00010000:1.0:1787827153.856608:0:6142:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: MGC192.168.203.129@tcp lock: ffff8cb981674c00/0x218c3ba1346961b lrc: 2/0,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x1400000000000 nid: local remote: 0xb2e8bd65d71dd9f7 expref: -99 pid: 6141 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787827153.856696:0:6142:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:1.0:1787827153.856702:0:6142:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff8cb999c6a940 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:0bb8b105-f59c-455a-b6b1-ce40efc3999c+4:10297:x1874672580865024:12345-192.168.203.129@tcp:104: Request processed in 111us (873us total) trans 0 rc 0/0 00000100:00100000:1.0:1787827153.856709:0:6142:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.129@tcp, seq: 0 00000100:00100000:1.0:1787827153.856726:0:6142:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.129@tcp, seq: 1 00000100:00100000:1.0:1787827153.856734:0:6142:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff8cb999c68b40 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:0bb8b105-f59c-455a-b6b1-ce40efc3999c+3:10297:x1874672580865280:12345-192.168.203.129@tcp:104: 00010000:00010000:2.0:1787827153.856739:0:6153:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:1.0:1787827153.856741:0:6142:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: MGC192.168.203.129@tcp lock: ffff8cb981674600/0x218c3ba13469614 lrc: 2/0,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x1400000000000 nid: local remote: 0xb2e8bd65d71dd9f0 expref: -99 pid: 6141 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1787827153.856747:0:6153:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: MGC192.168.203.129@tcp lock: ffff8cb981674c00/0x218c3ba1346961b lrc: 2/0,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 4 type: PLN flags: 0x1400000000000 nid: local remote: 0xb2e8bd65d71dd9f7 expref: -99 pid: 6141 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1787827153.856758:0:6153:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8cb981674c00 already unused, calling callback (ffffffffc132dc20) 10000000:00010000:2.0:1787827153.856762:0:6153:0:(mgc_request.c:922:mgc_blocking_ast()) ### MGC blocking CB ns: MGC192.168.203.129@tcp lock: ffff8cb981674c00/0x218c3ba1346961b lrc: 2/0,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x1400400000000 nid: local remote: 0xb2e8bd65d71dd9f7 expref: -99 pid: 6141 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1787827153.856782:0:6153:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: MGC192.168.203.129@tcp lock: ffff8cb981674c00/0x218c3ba1346961b lrc: 3/0,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x1408400000000 nid: local remote: 0xb2e8bd65d71dd9f7 expref: -99 pid: 6141 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:00010000:2.0:1787827153.856789:0:6153:0:(mgc_request.c:928:mgc_blocking_ast()) ### MGC cancel CB ns: MGC192.168.203.129@tcp lock: ffff8cb981674c00/0x218c3ba1346961b lrc: 3/0,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x1409400000000 nid: local remote: 0xb2e8bd65d71dd9f7 expref: -99 pid: 6141 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:01000000:2.0:1787827153.856795:0:6153:0:(mgc_request.c:932:mgc_blocking_ast()) Lock res [0x65727473756c:0x0:0x0].0x0 (lustre) 00010000:00010000:1.0:1787827153.856821:0:6142:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:1.0:1787827153.856834:0:6142:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff8cb999c68b40 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:0bb8b105-f59c-455a-b6b1-ce40efc3999c+3:10297:x1874672580865280:12345-192.168.203.129@tcp:104: Request processed in 96us (720us total) trans 0 rc 0/0 00000100:00100000:1.0:1787827153.856843:0:6142:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.129@tcp, seq: 1 00010000:00010000:2.0:1787827153.856861:0:6153:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: MGC192.168.203.129@tcp lock: ffff8cb981674c00/0x218c3ba1346961b lrc: 2/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x5c09400000020 nid: local remote: 0xb2e8bd65d71dd9f7 expref: -99 pid: 6141 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1787827153.856868:0:6153:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:1.0:1787827153.856883:0:6152:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:2.0:1787827153.856908:0:6153:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: MGC192.168.203.129@tcp lock: ffff8cb981674c00/0x218c3ba1346961b lrc: 1/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x5c09400000020 nid: local remote: 0xb2e8bd65d71dd9f7 expref: -99 pid: 6141 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1787827153.856917:0:6153:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: MGC192.168.203.129@tcp lock: ffff8cb981674c00/0x218c3ba1346961b lrc: 0/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x5c09400000020 nid: local remote: 0xb2e8bd65d71dd9f7 expref: -99 pid: 6141 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1787827153.856935:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787827153.856937:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787827153.856939:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787827153.856958:0:2396:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f7f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2396:1874672572436352:192.168.203.129@tcp:103: 00010000:00010000:1.0:1787827153.857120:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787827153.857124:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787827153.857127:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787827153.857361:0:6152:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: MGC192.168.203.129@tcp lock: ffff8cb981674600/0x218c3ba13469614 lrc: 2/0,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1400000000000 nid: local remote: 0xb2e8bd65d71dd9f0 expref: -99 pid: 6141 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1787827153.857369:0:6152:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8cb981674600 already unused, calling callback (ffffffffc132dc20) 10000000:00010000:2.0:1787827153.857373:0:6152:0:(mgc_request.c:922:mgc_blocking_ast()) ### MGC blocking CB ns: MGC192.168.203.129@tcp lock: ffff8cb981674600/0x218c3ba13469614 lrc: 2/0,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1400400000000 nid: local remote: 0xb2e8bd65d71dd9f0 expref: -99 pid: 6141 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1787827153.857380:0:6152:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: MGC192.168.203.129@tcp lock: ffff8cb981674600/0x218c3ba13469614 lrc: 3/0,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1408400000000 nid: local remote: 0xb2e8bd65d71dd9f0 expref: -99 pid: 6141 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:00010000:2.0:1787827153.857394:0:6152:0:(mgc_request.c:928:mgc_blocking_ast()) ### MGC cancel CB ns: MGC192.168.203.129@tcp lock: ffff8cb981674600/0x218c3ba13469614 lrc: 3/0,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1409400000000 nid: local remote: 0xb2e8bd65d71dd9f0 expref: -99 pid: 6141 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:01000000:2.0:1787827153.857404:0:6152:0:(mgc_request.c:932:mgc_blocking_ast()) Lock res [0x65727473756c:0x0:0x0].0x0 (lustre) 00010000:00010000:2.0:1787827153.857450:0:6152:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: MGC192.168.203.129@tcp lock: ffff8cb981674600/0x218c3ba13469614 lrc: 2/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x5c09400000020 nid: local remote: 0xb2e8bd65d71dd9f0 expref: -99 pid: 6141 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1787827153.857457:0:6152:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:2.0:1787827153.857471:0:6152:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: MGC192.168.203.129@tcp lock: ffff8cb981674600/0x218c3ba13469614 lrc: 1/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x5c09400000020 nid: local remote: 0xb2e8bd65d71dd9f0 expref: -99 pid: 6141 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1787827153.857478:0:6152:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: MGC192.168.203.129@tcp lock: ffff8cb981674600/0x218c3ba13469614 lrc: 0/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x5c09400000020 nid: local remote: 0xb2e8bd65d71dd9f0 expref: -99 pid: 6141 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1787827153.857503:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787827153.857506:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787827153.857508:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787827153.857521:0:2397:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8fc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2397:1874672572436480:192.168.203.129@tcp:103: 00000100:00100000:3.0:1787827153.858023:0:2396:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f7f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2396:1874672572436352:192.168.203.129@tcp:103: 00000100:00100000:2.0:1787827153.858596:0:2397:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8fc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2397:1874672572436480:192.168.203.129@tcp:103: 00010000:00010000:2.0:1787827154.114558:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787827155.138494:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 10000000:01000000:3.0:1787827155.202369:0:6155:0:(mgc_request.c:611:do_requeue()) updating log lustre-client 10000000:01000000:3.0:1787827155.202377:0:6155:0:(mgc_request.c:1802:mgc_process_log()) Process log lustre-client-ffff8cb98543f000 from 48 10000000:01000000:3.0:1787827155.202382:0:6155:0:(mgc_request.c:996:mgc_enqueue()) Enqueue for lustre-client (res 0x65727473756c) 00010000:00010000:3.0:1787827155.202425:0:6155:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: MGC192.168.203.129@tcp lock: ffff8cb981675800/0x218c3ba1346968b lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 6155 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787827155.202449:0:6155:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000000000000 ns: MGC192.168.203.129@tcp lock: ffff8cb981675800/0x218c3ba1346968b lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 6155 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787827155.202460:0:6155:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: MGC192.168.203.129@tcp lock: ffff8cb981675800/0x218c3ba1346968b lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 6155 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1787827155.202485:0:6155:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb992316580 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:0bb8b105-f59c-455a-b6b1-ce40efc3999c:6155:1874672572436608:192.168.203.129@tcp:101: 00000100:00100000:3.0:1787827155.202600:0:6155:0:(client.c:2715:ptlrpc_set_wait()) set ffff8cb990e8ea80 going to sleep for 11 seconds 00000100:00100000:3.0:1787827155.204037:0:6155:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb992316580 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:0bb8b105-f59c-455a-b6b1-ce40efc3999c:6155:1874672572436608:192.168.203.129@tcp:101: 00010000:00010000:3.0:1787827155.204059:0:6155:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: MGC192.168.203.129@tcp lock: ffff8cb981675800/0x218c3ba1346968b lrc: 4/1,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x11000000000000 nid: local remote: 0xb2e8bd65d71ddaf3 expref: -99 pid: 6155 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787827155.204083:0:6155:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: MGC192.168.203.129@tcp lock: ffff8cb981675800/0x218c3ba1346968b lrc: 4/1,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0xb2e8bd65d71ddaf3 expref: -99 pid: 6155 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1787827155.204137:0:6155:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb992316580 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:0bb8b105-f59c-455a-b6b1-ce40efc3999c:6155:1874672572436736:192.168.203.129@tcp:501: 00000100:00100000:3.0:1787827155.204184:0:6155:0:(client.c:2715:ptlrpc_set_wait()) set ffff8cb990e8ea80 going to sleep for 11 seconds 00000100:00100000:3.0:1787827155.205675:0:6155:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb992316580 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:0bb8b105-f59c-455a-b6b1-ce40efc3999c:6155:1874672572436736:192.168.203.129@tcp:501: 00000100:00100000:3.0:1787827155.205711:0:6155:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb992316580 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:0bb8b105-f59c-455a-b6b1-ce40efc3999c:6155:1874672572436864:192.168.203.129@tcp:503: 00000100:00100000:3.0:1787827155.205772:0:6155:0:(client.c:2715:ptlrpc_set_wait()) set ffff8cb990e8ea80 going to sleep for 11 seconds 00000100:00100000:3.0:1787827155.206970:0:6155:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb992316580 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:0bb8b105-f59c-455a-b6b1-ce40efc3999c:6155:1874672572436864:192.168.203.129@tcp:503: 00000100:00100000:3.0:1787827155.207350:0:6633:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb992316580 pname:cluuid:pid:xid:nid:opc:job llog_process_th:0bb8b105-f59c-455a-b6b1-ce40efc3999c:6633:1874672572436992:192.168.203.129@tcp:502: 00000100:00100000:3.0:1787827155.207405:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff8cb990e8e840 going to sleep for 11 seconds 00000100:00100000:3.0:1787827155.208553:0:6633:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb992316580 pname:cluuid:pid:xid:nid:opc:job llog_process_th:0bb8b105-f59c-455a-b6b1-ce40efc3999c:6633:1874672572436992:192.168.203.129@tcp:502: 00000020:01000000:3.0:1787827155.208581:0:6633:0:(obd_config.c:1893:class_config_llog_handler()) Marker, inst_flg=0x0 mark_flg=0x1 00000020:00000080:3.0:1787827155.208589:0:6633:0:(obd_config.c:1472:class_process_config()) processing cmd: cf010 00000020:00000080:3.0:1787827155.208591:0:6633:0:(obd_config.c:1563:class_process_config()) marker 41 (0x1) lustre sys.jobid_var 00000020:00000080:3.0:1787827155.208594:0:6633:0:(obd_config.c:1472:class_process_config()) processing cmd: cf00f 00000100:00100000:3.0:1787827155.208614:0:6633:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb992316580 pname:cluuid:pid:xid:nid:opc:job llog_process_th:0bb8b105-f59c-455a-b6b1-ce40efc3999c:6633:1874672572437120:192.168.203.129@tcp:502:llog_process_th.0 00000100:00100000:3.0:1787827155.208655:0:6633:0:(client.c:2715:ptlrpc_set_wait()) set ffff8cb990e8e840 going to sleep for 11 seconds 00000100:00100000:3.0:1787827155.209905:0:6633:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb992316580 pname:cluuid:pid:xid:nid:opc:job llog_process_th:0bb8b105-f59c-455a-b6b1-ce40efc3999c:6633:1874672572437120:192.168.203.129@tcp:502:llog_process_th.0 00000020:01000000:3.0:1787827155.209930:0:6633:0:(obd_config.c:1893:class_config_llog_handler()) Marker, inst_flg=0x2 mark_flg=0x2 00000020:00000080:3.0:1787827155.209936:0:6633:0:(obd_config.c:1472:class_process_config()) processing cmd: cf010 00000020:00000080:3.0:1787827155.209938:0:6633:0:(obd_config.c:1563:class_process_config()) marker 41 (0x2) lustre sys.jobid_var 00000040:00080000:3.0:1787827155.209941:0:6633:0:(llog.c:862:llog_process_thread()) stop processing plain [0xa:0x3:0x0] index 52 count 51 00000020:01000000:0.0:1787827155.210025:0:6155:0:(obd_config.c:2145:class_config_parse_llog()) Processed log lustre-client gen 48-51 (rc=0) 10000000:01000000:0.0:1787827155.210041:0:6155:0:(mgc_request.c:1879:mgc_process_log()) MGC192.168.203.129@tcp: configuration from log 'lustre-client' succeeded (0). 00010000:00010000:0.0:1787827155.210050:0:6155:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: MGC192.168.203.129@tcp lock: ffff8cb981675800/0x218c3ba1346968b lrc: 3/1,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x11000000000000 nid: local remote: 0xb2e8bd65d71ddaf3 expref: -99 pid: 6155 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787827155.210058:0:6155:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: MGC192.168.203.129@tcp lock: ffff8cb981675800/0x218c3ba1346968b lrc: 2/0,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x11000000000000 nid: local remote: 0xb2e8bd65d71ddaf3 expref: -99 pid: 6155 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:01000000:0.0:1787827155.210065:0:6155:0:(mgc_request.c:611:do_requeue()) updating log lustre-sptlrpc 10000000:01000000:0.0:1787827155.210071:0:6155:0:(mgc_request.c:1802:mgc_process_log()) Process log lustre-sptlrpc-ffff8cb9881c0000 from 1 10000000:01000000:0.0:1787827155.210073:0:6155:0:(mgc_request.c:996:mgc_enqueue()) Enqueue for lustre-sptlrpc (res 0x65727473756c) 00010000:00010000:0.0:1787827155.210094:0:6155:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: MGC192.168.203.129@tcp lock: ffff8cb980030a00/0x218c3ba13469692 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 6155 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787827155.210099:0:6155:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000000000000 ns: MGC192.168.203.129@tcp lock: ffff8cb980030a00/0x218c3ba13469692 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 6155 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787827155.210105:0:6155:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: MGC192.168.203.129@tcp lock: ffff8cb980030a00/0x218c3ba13469692 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 6155 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1787827155.210117:0:6155:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c6a1c0 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:0bb8b105-f59c-455a-b6b1-ce40efc3999c:6155:1874672572437248:192.168.203.129@tcp:101:ll_cfg_requeue.0 00000100:00100000:0.0:1787827155.210163:0:6155:0:(client.c:2715:ptlrpc_set_wait()) set ffff8cb988d76300 going to sleep for 11 seconds 00000100:00100000:3.0:1787827155.211528:0:6155:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c6a1c0 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:0bb8b105-f59c-455a-b6b1-ce40efc3999c:6155:1874672572437248:192.168.203.129@tcp:101:ll_cfg_requeue.0 00010000:00010000:3.0:1787827155.211549:0:6155:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: MGC192.168.203.129@tcp lock: ffff8cb980030a00/0x218c3ba13469692 lrc: 4/1,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x11000000000000 nid: local remote: 0xb2e8bd65d71ddafa expref: -99 pid: 6155 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787827155.211560:0:6155:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: MGC192.168.203.129@tcp lock: ffff8cb980030a00/0x218c3ba13469692 lrc: 4/1,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x1000000000000 nid: local remote: 0xb2e8bd65d71ddafa expref: -99 pid: 6155 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1787827155.211636:0:6155:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb992316580 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:0bb8b105-f59c-455a-b6b1-ce40efc3999c:6155:1874672572437376:192.168.203.129@tcp:501:ll_cfg_requeue.0 00000100:00100000:3.0:1787827155.211699:0:6155:0:(client.c:2715:ptlrpc_set_wait()) set ffff8cb990e8e840 going to sleep for 11 seconds 00000100:00100000:3.0:1787827155.212741:0:6155:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb992316580 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:0bb8b105-f59c-455a-b6b1-ce40efc3999c:6155:1874672572437376:192.168.203.129@tcp:501:ll_cfg_requeue.0 10000000:01000000:3.0:1787827155.212766:0:6155:0:(mgc_request.c:1879:mgc_process_log()) MGC192.168.203.129@tcp: configuration from log 'lustre-sptlrpc' failed (-2). 00010000:00010000:3.0:1787827155.212772:0:6155:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: MGC192.168.203.129@tcp lock: ffff8cb980030a00/0x218c3ba13469692 lrc: 3/1,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x11000000000000 nid: local remote: 0xb2e8bd65d71ddafa expref: -99 pid: 6155 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787827155.212780:0:6155:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: MGC192.168.203.129@tcp lock: ffff8cb980030a00/0x218c3ba13469692 lrc: 2/0,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x11000000000000 nid: local remote: 0xb2e8bd65d71ddafa expref: -99 pid: 6155 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1787827155.330523:0:2396:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb992316580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2396:1874672572437504:192.168.203.129@tcp:400:kworker.0 00000100:00100000:3.0:1787827155.330534:0:2397:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb9923170c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2397:1874672572437632:192.168.203.129@tcp:400:kworker.0 00000100:00100000:2.0:1787827155.330610:0:2396:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb9923161c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2396:1874672572437760:192.168.203.129@tcp:400:kworker.0 00000100:00100000:3.0:1787827155.330622:0:2397:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb9923143c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2397:1874672572437888:192.168.203.129@tcp:400:kworker.0 00000100:00100000:2.0:1787827155.332147:0:2396:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb992316580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2396:1874672572437504:192.168.203.129@tcp:400:kworker.0 00010000:00080000:3.0:1787827155.332173:0:2397:0:(ldlm_request.c:1671:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=46162) req@ffff8cb9923170c0 x1874672572437632/t0(0) o400->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 224/224 e 0 to 0 dl 1787827171 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00010000:00080000:2.0:1787827155.332177:0:2396:0:(ldlm_request.c:1671:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=46162) req@ffff8cb9923161c0 x1874672572437760/t0(0) o400->lustre-OST0000-osc-ffff8cb98543f000@192.168.203.129@tcp:28/4 lens 224/224 e 0 to 0 dl 1787827171 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827155.332188:0:2397:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb9923170c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2397:1874672572437632:192.168.203.129@tcp:400:kworker.0 00010000:00080000:3.0:1787827155.332200:0:2397:0:(ldlm_request.c:1671:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=46162) req@ffff8cb9923143c0 x1874672572437888/t0(0) o400->lustre-OST0001-osc-ffff8cb98543f000@192.168.203.129@tcp:28/4 lens 224/224 e 0 to 0 dl 1787827171 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827155.332223:0:2397:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb9923143c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2397:1874672572437888:192.168.203.129@tcp:400:kworker.0 00000100:00100000:2.0:1787827155.332226:0:2396:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb9923161c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2396:1874672572437760:192.168.203.129@tcp:400:kworker.0 00010000:00010000:1.0:1787827156.162365:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787827156.281962:0:2387:0:(events.c:340:request_in_callback()) peer: 12345-192.168.203.129@tcp (source: 12345-192.168.203.129@tcp) 00000100:00100000:0.0:1787827156.283504:0:6142:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1874672580868480 00000100:00100000:0.0:1787827156.283516:0:6142:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1874672580868480 00000100:00100000:0.0:1787827156.283588:0:6142:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.129@tcp, seq: 2 00000100:00100000:0.0:1787827156.283598:0:6142:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff8cb999f3bc00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:0bb8b105-f59c-455a-b6b1-ce40efc3999c+3:10352:x1874672580868480:12345-192.168.203.129@tcp:104: 00010000:00010000:0.0:1787827156.283615:0:6142:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: MGC192.168.203.129@tcp lock: ffff8cb981674800/0x218c3ba13469653 lrc: 2/0,0 mode: CR/CR res: [0x736d61726170:0x3:0x0].0x0 rrc: 2 type: PLN flags: 0x1400000000000 nid: local remote: 0xb2e8bd65d71dda05 expref: -99 pid: 6141 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787827156.283698:0:6142:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:0.0:1787827156.283705:0:6142:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff8cb999f3bc00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:0bb8b105-f59c-455a-b6b1-ce40efc3999c+3:10352:x1874672580868480:12345-192.168.203.129@tcp:104: Request processed in 109us (1751us total) trans 0 rc 0/0 00000100:00100000:0.0:1787827156.283713:0:6142:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.129@tcp, seq: 2 00010000:00010000:2.0:1787827156.283792:0:6153:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:2.0:1787827156.283800:0:6153:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: MGC192.168.203.129@tcp lock: ffff8cb981674800/0x218c3ba13469653 lrc: 2/0,0 mode: CR/CR res: [0x736d61726170:0x3:0x0].0x0 rrc: 2 type: PLN flags: 0x1400000000000 nid: local remote: 0xb2e8bd65d71dda05 expref: -99 pid: 6141 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1787827156.283809:0:6153:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8cb981674800 already unused, calling callback (ffffffffc132dc20) 10000000:00010000:2.0:1787827156.283818:0:6153:0:(mgc_request.c:922:mgc_blocking_ast()) ### MGC blocking CB ns: MGC192.168.203.129@tcp lock: ffff8cb981674800/0x218c3ba13469653 lrc: 2/0,0 mode: CR/CR res: [0x736d61726170:0x3:0x0].0x0 rrc: 2 type: PLN flags: 0x1400400000000 nid: local remote: 0xb2e8bd65d71dda05 expref: -99 pid: 6141 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1787827156.283828:0:6153:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: MGC192.168.203.129@tcp lock: ffff8cb981674800/0x218c3ba13469653 lrc: 3/0,0 mode: CR/CR res: [0x736d61726170:0x3:0x0].0x0 rrc: 2 type: PLN flags: 0x1408400000000 nid: local remote: 0xb2e8bd65d71dda05 expref: -99 pid: 6141 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:00010000:2.0:1787827156.283836:0:6153:0:(mgc_request.c:928:mgc_blocking_ast()) ### MGC cancel CB ns: MGC192.168.203.129@tcp lock: ffff8cb981674800/0x218c3ba13469653 lrc: 3/0,0 mode: CR/CR res: [0x736d61726170:0x3:0x0].0x0 rrc: 2 type: PLN flags: 0x1409400000000 nid: local remote: 0xb2e8bd65d71dda05 expref: -99 pid: 6141 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:01000000:2.0:1787827156.283842:0:6153:0:(mgc_request.c:932:mgc_blocking_ast()) Lock res [0x736d61726170:0x3:0x0].0x0 (params) 00010000:00010000:2.0:1787827156.283877:0:6153:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: MGC192.168.203.129@tcp lock: ffff8cb981674800/0x218c3ba13469653 lrc: 2/0,0 mode: --/CR res: [0x736d61726170:0x3:0x0].0x0 rrc: 2 type: PLN flags: 0x5c09400000020 nid: local remote: 0xb2e8bd65d71dda05 expref: -99 pid: 6141 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1787827156.283883:0:6153:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:2.0:1787827156.283903:0:6153:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: MGC192.168.203.129@tcp lock: ffff8cb981674800/0x218c3ba13469653 lrc: 1/0,0 mode: --/CR res: [0x736d61726170:0x3:0x0].0x0 rrc: 2 type: PLN flags: 0x5c09400000020 nid: local remote: 0xb2e8bd65d71dda05 expref: -99 pid: 6141 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1787827156.283910:0:6153:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: MGC192.168.203.129@tcp lock: ffff8cb981674800/0x218c3ba13469653 lrc: 0/0,0 mode: --/CR res: [0x736d61726170:0x3:0x0].0x0 rrc: 2 type: PLN flags: 0x5c09400000020 nid: local remote: 0xb2e8bd65d71dda05 expref: -99 pid: 6141 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1787827156.283929:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787827156.283931:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787827156.283934:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827156.283945:0:2397:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:3.0:1787827156.283966:0:2397:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8fc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2397:1874672572438016:192.168.203.129@tcp:103:ldlm_bl.0 00000100:00100000:3.0:1787827156.285086:0:2397:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8fc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2397:1874672572438016:192.168.203.129@tcp:103:ldlm_bl.0 10000000:01000000:3.0:1787827156.930365:0:6155:0:(mgc_request.c:611:do_requeue()) updating log params 10000000:01000000:3.0:1787827156.930373:0:6155:0:(mgc_request.c:1802:mgc_process_log()) Process log params-ffff8cb98543f000 from 4 10000000:01000000:3.0:1787827156.930377:0:6155:0:(mgc_request.c:996:mgc_enqueue()) Enqueue for params (res 0x736d61726170) 00010000:00010000:3.0:1787827156.930434:0:6155:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: MGC192.168.203.129@tcp lock: ffff8cb981675200/0x218c3ba13469699 lrc: 3/1,0 mode: --/CR res: [0x736d61726170:0x3:0x0].0x0 rrc: 2 type: PLN flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 6155 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787827156.930446:0:6155:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000000000000 ns: MGC192.168.203.129@tcp lock: ffff8cb981675200/0x218c3ba13469699 lrc: 3/1,0 mode: --/CR res: [0x736d61726170:0x3:0x0].0x0 rrc: 2 type: PLN flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 6155 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787827156.930457:0:6155:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: MGC192.168.203.129@tcp lock: ffff8cb981675200/0x218c3ba13469699 lrc: 3/1,0 mode: --/CR res: [0x736d61726170:0x3:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 6155 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1787827156.930482:0:6155:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb9923170c0 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:0bb8b105-f59c-455a-b6b1-ce40efc3999c:6155:1874672572438144:192.168.203.129@tcp:101:ll_cfg_requeue.0 00000100:00100000:3.0:1787827156.930572:0:6155:0:(client.c:2715:ptlrpc_set_wait()) set ffff8cb990e8e600 going to sleep for 11 seconds 00000100:00100000:3.0:1787827156.932214:0:6155:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb9923170c0 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:0bb8b105-f59c-455a-b6b1-ce40efc3999c:6155:1874672572438144:192.168.203.129@tcp:101:ll_cfg_requeue.0 00010000:00010000:3.0:1787827156.932239:0:6155:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: MGC192.168.203.129@tcp lock: ffff8cb981675200/0x218c3ba13469699 lrc: 4/1,0 mode: CR/CR res: [0x736d61726170:0x3:0x0].0x0 rrc: 2 type: PLN flags: 0x11000000000000 nid: local remote: 0xb2e8bd65d71ddb08 expref: -99 pid: 6155 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787827156.932250:0:6155:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: MGC192.168.203.129@tcp lock: ffff8cb981675200/0x218c3ba13469699 lrc: 4/1,0 mode: CR/CR res: [0x736d61726170:0x3:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0xb2e8bd65d71ddb08 expref: -99 pid: 6155 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1787827156.932334:0:6155:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb9923170c0 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:0bb8b105-f59c-455a-b6b1-ce40efc3999c:6155:1874672572438272:192.168.203.129@tcp:501:ll_cfg_requeue.0 00000100:00100000:3.0:1787827156.932381:0:6155:0:(client.c:2715:ptlrpc_set_wait()) set ffff8cb990e8e600 going to sleep for 11 seconds 00000100:00100000:3.0:1787827156.933378:0:6155:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb9923170c0 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:0bb8b105-f59c-455a-b6b1-ce40efc3999c:6155:1874672572438272:192.168.203.129@tcp:501:ll_cfg_requeue.0 00000100:00100000:3.0:1787827156.933410:0:6155:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb9923170c0 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:0bb8b105-f59c-455a-b6b1-ce40efc3999c:6155:1874672572438400:192.168.203.129@tcp:503:ll_cfg_requeue.0 00000100:00100000:3.0:1787827156.933474:0:6155:0:(client.c:2715:ptlrpc_set_wait()) set ffff8cb990e8e600 going to sleep for 11 seconds 00000100:00100000:3.0:1787827156.934773:0:6155:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb9923170c0 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:0bb8b105-f59c-455a-b6b1-ce40efc3999c:6155:1874672572438400:192.168.203.129@tcp:503:ll_cfg_requeue.0 00000100:00100000:3.0:1787827156.935148:0:6696:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb9923170c0 pname:cluuid:pid:xid:nid:opc:job llog_process_th:0bb8b105-f59c-455a-b6b1-ce40efc3999c:6696:1874672572438528:192.168.203.129@tcp:502:llog_process_th.0 00000100:00100000:3.0:1787827156.935204:0:6696:0:(client.c:2715:ptlrpc_set_wait()) set ffff8cb990e8ea80 going to sleep for 11 seconds 00000100:00100000:3.0:1787827156.936480:0:6696:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb9923170c0 pname:cluuid:pid:xid:nid:opc:job llog_process_th:0bb8b105-f59c-455a-b6b1-ce40efc3999c:6696:1874672572438528:192.168.203.129@tcp:502:llog_process_th.0 00000020:01000000:3.0:1787827156.936507:0:6696:0:(obd_config.c:1893:class_config_llog_handler()) Marker, inst_flg=0x0 mark_flg=0x1 00000020:00000080:3.0:1787827156.936516:0:6696:0:(obd_config.c:1472:class_process_config()) processing cmd: cf010 00000020:00000080:3.0:1787827156.936518:0:6696:0:(obd_config.c:1563:class_process_config()) marker 3 (0x1) general lod.*.mdt_hash 00000020:01000000:3.0:1787827156.936524:0:6696:0:(obd_config.c:2004:class_config_llog_handler()) cmd ce032, instance name: general-ffff8cb98543f000 00000020:00000080:3.0:1787827156.936527:0:6696:0:(obd_config.c:1472:class_process_config()) processing cmd: ce032 00000100:00100000:3.0:1787827156.936592:0:6696:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb9923170c0 pname:cluuid:pid:xid:nid:opc:job llog_process_th:0bb8b105-f59c-455a-b6b1-ce40efc3999c:6696:1874672572438656:192.168.203.129@tcp:502:llog_process_th.0 00000100:00100000:3.0:1787827156.936657:0:6696:0:(client.c:2715:ptlrpc_set_wait()) set ffff8cb990e8ea80 going to sleep for 11 seconds 00000100:00100000:3.0:1787827156.937736:0:6696:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb9923170c0 pname:cluuid:pid:xid:nid:opc:job llog_process_th:0bb8b105-f59c-455a-b6b1-ce40efc3999c:6696:1874672572438656:192.168.203.129@tcp:502:llog_process_th.0 00000020:01000000:3.0:1787827156.937785:0:6696:0:(obd_config.c:1893:class_config_llog_handler()) Marker, inst_flg=0x2 mark_flg=0x2 00000020:00000080:3.0:1787827156.937796:0:6696:0:(obd_config.c:1472:class_process_config()) processing cmd: cf010 00000020:00000080:3.0:1787827156.937798:0:6696:0:(obd_config.c:1563:class_process_config()) marker 3 (0x2) general lod.*.mdt_hash 00000040:00080000:3.0:1787827156.937802:0:6696:0:(llog.c:862:llog_process_thread()) stop processing plain [0xa:0x2:0x0] index 7 count 7 00000020:01000000:0.0:1787827156.937902:0:6155:0:(obd_config.c:2145:class_config_parse_llog()) Processed log params gen 4-6 (rc=0) 10000000:01000000:0.0:1787827156.937937:0:6155:0:(mgc_request.c:1879:mgc_process_log()) MGC192.168.203.129@tcp: configuration from log 'params' succeeded (0). 00010000:00010000:0.0:1787827156.937949:0:6155:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: MGC192.168.203.129@tcp lock: ffff8cb981675200/0x218c3ba13469699 lrc: 3/1,0 mode: CR/CR res: [0x736d61726170:0x3:0x0].0x0 rrc: 2 type: PLN flags: 0x11000000000000 nid: local remote: 0xb2e8bd65d71ddb08 expref: -99 pid: 6155 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787827156.937973:0:6155:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: MGC192.168.203.129@tcp lock: ffff8cb981675200/0x218c3ba13469699 lrc: 2/0,0 mode: CR/CR res: [0x736d61726170:0x3:0x0].0x0 rrc: 2 type: PLN flags: 0x11000000000000 nid: local remote: 0xb2e8bd65d71ddb08 expref: -99 pid: 6155 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787827157.187764:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787827158.210443:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000001:02000400:1.0:1787827158.517545:0:6995:0:(debug.c:653:libcfs_debug_mark_buffer()) DEBUG MARKER: oleg329-client.virtnet: executing check_logdir /tmp/testlogs/ 00010000:00010000:0.0:1787827159.234375:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000001:02000400:2.0:1787827160.173536:0:7274:0:(debug.c:653:libcfs_debug_mark_buffer()) DEBUG MARKER: oleg329-client.virtnet: executing yml_node 00010000:00010000:0.0:1787827160.259913:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787827160.451435:0:2397:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8d680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2397:1874672572438784:192.168.203.129@tcp:400:kworker.0 00000100:00100000:2.0:1787827160.451534:0:2397:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8cf00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2397:1874672572439040:192.168.203.129@tcp:400:kworker.0 00000100:00100000:2.0:1787827160.451571:0:2397:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8e1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2397:1874672572439296:192.168.203.129@tcp:400:kworker.0 00000100:00100000:3.0:1787827160.451579:0:2396:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8de00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2396:1874672572438912:192.168.203.129@tcp:400:kworker.0 00000100:00100000:3.0:1787827160.451623:0:2396:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8c3c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2396:1874672572439168:192.168.203.129@tcp:400:kworker.0 00000100:00100000:2.0:1787827160.452953:0:2397:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8d680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2397:1874672572438784:192.168.203.129@tcp:400:kworker.0 00010000:00080000:2.0:1787827160.452970:0:2397:0:(ldlm_request.c:1671:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=46162) req@ffff8cb998a8cf00 x1874672572439040/t0(0) o400->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 224/224 e 0 to 0 dl 1787827176 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787827160.452983:0:2397:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8cf00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2397:1874672572439040:192.168.203.129@tcp:400:kworker.0 00010000:00080000:2.0:1787827160.453238:0:2397:0:(ldlm_request.c:1671:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=46162) req@ffff8cb998a8e1c0 x1874672572439296/t0(0) o400->lustre-OST0001-osc-ffff8cb98543f000@192.168.203.129@tcp:28/4 lens 224/224 e 0 to 0 dl 1787827176 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787827160.453250:0:2397:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8e1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2397:1874672572439296:192.168.203.129@tcp:400:kworker.0 00000100:00100000:3.0:1787827160.453314:0:2396:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8de00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2396:1874672572438912:192.168.203.129@tcp:400:kworker.0 00010000:00080000:3.0:1787827160.453326:0:2396:0:(ldlm_request.c:1671:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=46162) req@ffff8cb998a8c3c0 x1874672572439168/t0(0) o400->lustre-OST0000-osc-ffff8cb98543f000@192.168.203.129@tcp:28/4 lens 224/224 e 0 to 0 dl 1787827176 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827160.453336:0:2396:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8c3c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2396:1874672572439168:192.168.203.129@tcp:400:kworker.0 00010000:00010000:2.0:1787827161.283372:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000001:02000400:0.0:1787827161.893503:0:7728:0:(debug.c:653:libcfs_debug_mark_buffer()) DEBUG MARKER: Client: 2.17.57.65 00010000:00010000:0.0:1787827162.306464:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000001:02000400:1.0:1787827162.774239:0:7884:0:(debug.c:653:libcfs_debug_mark_buffer()) DEBUG MARKER: MDS: 2.17.57.65 00010000:00010000:0.0:1787827163.330367:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000001:02000400:2.0:1787827163.624924:0:8040:0:(debug.c:653:libcfs_debug_mark_buffer()) DEBUG MARKER: OSS: 2.17.57.65 00000001:02000400:2.0:1787827164.177423:0:8187:0:(debug.c:653:libcfs_debug_mark_buffer()) DEBUG MARKER: -----============= acceptance-small: sanity-sec ============----- Thu Aug 27 06:39:23 EDT 2026 00010000:00010000:2.0:1787827164.355021:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787827165.378393:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787827165.570555:0:2395:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c68f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2395:1874672572439424:192.168.203.129@tcp:400:kworker.0 00000100:00100000:0.0:1787827165.570565:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c69e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2394:1874672572439552:192.168.203.129@tcp:400:kworker.0 00000100:00100000:1.0:1787827165.570682:0:2395:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c69680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2395:1874672572439680:192.168.203.129@tcp:400:kworker.0 00000100:00100000:0.0:1787827165.570685:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c692c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2394:1874672572439808:192.168.203.129@tcp:400:kworker.0 00000100:00100000:1.0:1787827165.570721:0:2395:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c6bc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2395:1874672572439936:192.168.203.129@tcp:400:kworker.0 00000100:00100000:0.0:1787827165.571848:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c69e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2394:1874672572439552:192.168.203.129@tcp:400:kworker.0 00000100:00100000:0.0:1787827165.571902:0:2395:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c68f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2395:1874672572439424:192.168.203.129@tcp:400:kworker.0 00010000:00080000:0.0:1787827165.571923:0:2395:0:(ldlm_request.c:1671:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=46162) req@ffff8cb999c69680 x1874672572439680/t0(0) o400->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 224/224 e 0 to 0 dl 1787827181 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827165.571938:0:2395:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c69680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2395:1874672572439680:192.168.203.129@tcp:400:kworker.0 00010000:00080000:1.0:1787827165.571938:0:2394:0:(ldlm_request.c:1671:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=46162) req@ffff8cb999c692c0 x1874672572439808/t0(0) o400->lustre-OST0000-osc-ffff8cb98543f000@192.168.203.129@tcp:28/4 lens 224/224 e 0 to 0 dl 1787827181 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787827165.571955:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c692c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2394:1874672572439808:192.168.203.129@tcp:400:kworker.0 00010000:00080000:1.0:1787827165.572017:0:2395:0:(ldlm_request.c:1671:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=46162) req@ffff8cb999c6bc00 x1874672572439936/t0(0) o400->lustre-OST0001-osc-ffff8cb98543f000@192.168.203.129@tcp:28/4 lens 224/224 e 0 to 0 dl 1787827181 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787827165.572027:0:2395:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c6bc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2395:1874672572439936:192.168.203.129@tcp:400:kworker.0 00010000:00010000:0.0:1787827166.402426:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787827167.426410:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787827168.450509:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787827169.475360:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000001:02000400:1.0:1787827169.553710:0:8707:0:(debug.c:653:libcfs_debug_mark_buffer()) DEBUG MARKER: excepting tests: 27 66 00000001:02000400:1.0:1787827170.034863:0:8846:0:(debug.c:653:libcfs_debug_mark_buffer()) DEBUG MARKER: skipping tests SLOW=no: 26 00010000:00010000:3.0:1787827170.498352:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787827170.690494:0:2396:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb992315a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2396:1874672572440064:192.168.203.129@tcp:400:kworker.0 00000100:00100000:3.0:1787827170.690578:0:2397:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb992315e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2397:1874672572440192:192.168.203.129@tcp:400:kworker.0 00000100:00100000:2.0:1787827170.690580:0:2396:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb992314f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2396:1874672572440320:192.168.203.129@tcp:400:kworker.0 00000100:00100000:3.0:1787827170.690626:0:2397:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb9923143c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2397:1874672572440448:192.168.203.129@tcp:400:kworker.0 00000100:00100000:2.0:1787827170.690652:0:2396:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb9923161c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2396:1874672572440576:192.168.203.129@tcp:400:kworker.0 00000100:00100000:2.0:1787827170.691762:0:2396:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb992315a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2396:1874672572440064:192.168.203.129@tcp:400:kworker.0 00000100:00100000:2.0:1787827170.691921:0:2397:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb992315e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2397:1874672572440192:192.168.203.129@tcp:400:kworker.0 00010000:00080000:2.0:1787827170.692071:0:2396:0:(ldlm_request.c:1671:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=46162) req@ffff8cb9923161c0 x1874672572440576/t0(0) o400->lustre-OST0001-osc-ffff8cb98543f000@192.168.203.129@tcp:28/4 lens 224/224 e 0 to 0 dl 1787827186 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787827170.692084:0:2396:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb9923161c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2396:1874672572440576:192.168.203.129@tcp:400:kworker.0 00010000:00080000:2.0:1787827170.692096:0:2396:0:(ldlm_request.c:1671:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=46162) req@ffff8cb992314f00 x1874672572440320/t0(0) o400->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 224/224 e 0 to 0 dl 1787827186 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787827170.692106:0:2396:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb992314f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2396:1874672572440320:192.168.203.129@tcp:400:kworker.0 00010000:00080000:2.0:1787827170.692300:0:2397:0:(ldlm_request.c:1671:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=46162) req@ffff8cb9923143c0 x1874672572440448/t0(0) o400->lustre-OST0000-osc-ffff8cb98543f000@192.168.203.129@tcp:28/4 lens 224/224 e 0 to 0 dl 1787827186 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787827170.692313:0:2397:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb9923143c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2397:1874672572440448:192.168.203.129@tcp:400:kworker.0 00000001:02000400:3.0:1787827170.808752:0:9081:0:(debug.c:653:libcfs_debug_mark_buffer()) DEBUG MARKER: === sanity-sec: start setup 06:39:30 (1787827170) === 00000080:00200008:1.0:1787827170.820841:0:8201:0:(file.c:6313:ll_getattr_dentry()) START file /:[0x200000007:0x1:0x0](ffff8cb998d13a88), request_mask 2047, flags 0, foreign 0 00000080:00200000:1.0:1787827170.820853:0:8201:0:(file.c:6208:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8cb998d13a88),name=/ 00000002:00010000:1.0:1787827170.820875:0:8201:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1787827170.820892:0:8201:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8cb98543f000 lock: ffff8cb983df0e00/0x218c3ba13469684 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xb2e8bd65d71dda2f expref: -99 pid: 6180 timeout: 0 lvb_type: 0 lru_score: 4 lru_type: 1 00010000:00010000:1.0:1787827170.820907:0:8201:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8cb98543f000 lock: ffff8cb983df0e00/0x218c3ba13469684 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xb2e8bd65d71dda2f expref: -99 pid: 6180 timeout: 0 lvb_type: 0 lru_score: 4 lru_type: 1 00000080:00200000:1.0:1787827170.820927:0:8201:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1787827170.820936:0:8201:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8cb998d13a88) 00000080:00010000:1.0:1787827170.820940:0:8201:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8cb998d13a88) for lock 0x218c3ba13469684 00000080:00010000:1.0:1787827170.820947:0:8201:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x218c3ba13469684 from it ffffb8e3c1bfbcb8 00010000:00010000:1.0:1787827170.820951:0:8201:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8cb98543f000 lock: ffff8cb983df0e00/0x218c3ba13469684 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xb2e8bd65d71dda2f expref: -99 pid: 6180 timeout: 0 lvb_type: 0 lru_score: 4 lru_type: 1 00010000:00010000:1.0:1787827170.820960:0:8201:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8cb98543f000 lock: ffff8cb983df0e00/0x218c3ba13469684 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xb2e8bd65d71dda2f expref: -99 pid: 6180 timeout: 0 lvb_type: 0 lru_score: 5 lru_type: 1 00000080:00200008:1.0:1787827170.832628:0:8201:0:(file.c:6313:ll_getattr_dentry()) START file /:[0x200000007:0x1:0x0](ffff8cb998d13a88), request_mask 2047, flags 0, foreign 0 00000080:00200000:1.0:1787827170.832639:0:8201:0:(file.c:6208:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8cb998d13a88),name=/ 00000002:00010000:1.0:1787827170.832660:0:8201:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1787827170.832688:0:8201:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8cb98543f000 lock: ffff8cb983df0e00/0x218c3ba13469684 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xb2e8bd65d71dda2f expref: -99 pid: 6180 timeout: 0 lvb_type: 0 lru_score: 5 lru_type: 1 00010000:00010000:1.0:1787827170.832703:0:8201:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8cb98543f000 lock: ffff8cb983df0e00/0x218c3ba13469684 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xb2e8bd65d71dda2f expref: -99 pid: 6180 timeout: 0 lvb_type: 0 lru_score: 5 lru_type: 1 00000080:00200000:1.0:1787827170.832712:0:8201:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1787827170.832729:0:8201:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8cb998d13a88) 00000080:00010000:1.0:1787827170.832733:0:8201:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8cb998d13a88) for lock 0x218c3ba13469684 00000080:00010000:1.0:1787827170.832743:0:8201:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x218c3ba13469684 from it ffffb8e3c1bfbcb8 00010000:00010000:1.0:1787827170.832747:0:8201:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8cb98543f000 lock: ffff8cb983df0e00/0x218c3ba13469684 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xb2e8bd65d71dda2f expref: -99 pid: 6180 timeout: 0 lvb_type: 0 lru_score: 5 lru_type: 1 00010000:00010000:1.0:1787827170.832768:0:8201:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8cb98543f000 lock: ffff8cb983df0e00/0x218c3ba13469684 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xb2e8bd65d71dda2f expref: -99 pid: 6180 timeout: 0 lvb_type: 0 lru_score: 6 lru_type: 1 00000080:00200008:1.0:1787827170.842768:0:8201:0:(file.c:6313:ll_getattr_dentry()) START file /:[0x200000007:0x1:0x0](ffff8cb998d13a88), request_mask 2047, flags 0, foreign 0 00000080:00200000:1.0:1787827170.842776:0:8201:0:(file.c:6208:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8cb998d13a88),name=/ 00000002:00010000:1.0:1787827170.842794:0:8201:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1787827170.842808:0:8201:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8cb98543f000 lock: ffff8cb983df0e00/0x218c3ba13469684 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xb2e8bd65d71dda2f expref: -99 pid: 6180 timeout: 0 lvb_type: 0 lru_score: 6 lru_type: 1 00010000:00010000:1.0:1787827170.842818:0:8201:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8cb98543f000 lock: ffff8cb983df0e00/0x218c3ba13469684 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xb2e8bd65d71dda2f expref: -99 pid: 6180 timeout: 0 lvb_type: 0 lru_score: 6 lru_type: 1 00000080:00200000:1.0:1787827170.842823:0:8201:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1787827170.842829:0:8201:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8cb998d13a88) 00000080:00010000:1.0:1787827170.842831:0:8201:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8cb998d13a88) for lock 0x218c3ba13469684 00000080:00010000:1.0:1787827170.842837:0:8201:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x218c3ba13469684 from it ffffb8e3c1bfbcb8 00010000:00010000:1.0:1787827170.842839:0:8201:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8cb98543f000 lock: ffff8cb983df0e00/0x218c3ba13469684 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xb2e8bd65d71dda2f expref: -99 pid: 6180 timeout: 0 lvb_type: 0 lru_score: 6 lru_type: 1 00010000:00010000:1.0:1787827170.842851:0:8201:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8cb98543f000 lock: ffff8cb983df0e00/0x218c3ba13469684 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xb2e8bd65d71dda2f expref: -99 pid: 6180 timeout: 0 lvb_type: 0 lru_score: 7 lru_type: 1 00000080:00200008:1.0:1787827170.852756:0:8201:0:(file.c:6313:ll_getattr_dentry()) START file /:[0x200000007:0x1:0x0](ffff8cb998d13a88), request_mask 2047, flags 0, foreign 0 00000080:00200000:1.0:1787827170.852771:0:8201:0:(file.c:6208:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8cb998d13a88),name=/ 00000002:00010000:1.0:1787827170.852792:0:8201:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1787827170.852810:0:8201:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8cb98543f000 lock: ffff8cb983df0e00/0x218c3ba13469684 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xb2e8bd65d71dda2f expref: -99 pid: 6180 timeout: 0 lvb_type: 0 lru_score: 7 lru_type: 1 00010000:00010000:1.0:1787827170.852832:0:8201:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8cb98543f000 lock: ffff8cb983df0e00/0x218c3ba13469684 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xb2e8bd65d71dda2f expref: -99 pid: 6180 timeout: 0 lvb_type: 0 lru_score: 7 lru_type: 1 00000080:00200000:1.0:1787827170.852842:0:8201:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1787827170.852850:0:8201:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8cb998d13a88) 00000080:00010000:1.0:1787827170.852854:0:8201:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8cb998d13a88) for lock 0x218c3ba13469684 00000080:00010000:1.0:1787827170.852861:0:8201:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x218c3ba13469684 from it ffffb8e3c1bfbcb8 00010000:00010000:1.0:1787827170.852866:0:8201:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8cb98543f000 lock: ffff8cb983df0e00/0x218c3ba13469684 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xb2e8bd65d71dda2f expref: -99 pid: 6180 timeout: 0 lvb_type: 0 lru_score: 7 lru_type: 1 00010000:00010000:1.0:1787827170.852874:0:8201:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8cb98543f000 lock: ffff8cb983df0e00/0x218c3ba13469684 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xb2e8bd65d71dda2f expref: -99 pid: 6180 timeout: 0 lvb_type: 0 lru_score: 8 lru_type: 1 00010000:00010000:2.0:1787827171.522434:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787827172.546431:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000001:02000400:2.0:1787827172.666945:0:9350:0:(debug.c:653:libcfs_debug_mark_buffer()) DEBUG MARKER: oleg329-client.virtnet: executing check_config_client /mnt/lustre 00010000:00010000:3.0:1787827173.571331:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787827174.594401:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787827175.618446:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787827175.810525:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab1000 lustre-OST0000_UUID: changing import state from FULL to CONNECTING 00000100:02000000:2.0:1787827175.810533:0:36:0:(import.c:2101:ptlrpc_disconnect_and_idle_import()) lustre-OST0000-osc-ffff8cb98543f000: disconnect after 24s idle 00000100:00100000:3.0:1787827175.810555:0:2397:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8d680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2397:1874672572440704:192.168.203.129@tcp:400:kworker.0 00000100:00100000:3.0:1787827175.810674:0:2397:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8e1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2397:1874672572440960:192.168.203.129@tcp:400:kworker.0 00000100:00100000:3.0:1787827175.810790:0:2397:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:3.0:1787827175.810799:0:2397:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8cf00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2397:1874672572440832:192.168.203.129@tcp:400:kworker.0 00000100:00100000:3.0:1787827175.812101:0:2397:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8cf00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2397:1874672572440832:192.168.203.129@tcp:400:kworker.0 00000100:00100000:3.0:1787827175.812110:0:2397:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8d680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2397:1874672572440704:192.168.203.129@tcp:400:kworker.0 00010000:00080000:3.0:1787827175.812119:0:2397:0:(ldlm_request.c:1671:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=46162) req@ffff8cb998a8e1c0 x1874672572440960/t0(0) o400->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 224/224 e 0 to 0 dl 1787827191 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827175.812126:0:2397:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8e1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2397:1874672572440960:192.168.203.129@tcp:400:kworker.0 00000100:00080000:2.0:1787827175.814200:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab7800 lustre-OST0001_UUID: changing import state from FULL to CONNECTING 00000100:02000000:2.0:1787827175.814205:0:36:0:(import.c:2101:ptlrpc_disconnect_and_idle_import()) lustre-OST0001-osc-ffff8cb98543f000: disconnect after 24s idle 00000100:00100000:0.0:1787827175.814219:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8c3c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572441088:192.168.203.129@tcp:9:kworker.0 00000100:00100000:0.0:1787827175.814331:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8de00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572441216:192.168.203.129@tcp:9:kworker.0 00010000:00080000:2.0:1787827175.815393:0:2393:0:(ldlm_request.c:1671:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=46162) req@ffff8cb998a8de00 x1874672572441216/t0(0) o9->lustre-OST0001-osc-ffff8cb98543f000@192.168.203.129@tcp:28/4 lens 224/224 e 0 to 0 dl 1787827185 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787827175.815406:0:2393:0:(import.c:2011:ptlrpc_disconnect_idle_interpret()) @@@ inflight=1, refcount=3: rc = 0 req@ffff8cb998a8de00 x1874672572441216/t0(0) o9->lustre-OST0001-osc-ffff8cb98543f000@192.168.203.129@tcp:28/4 lens 224/224 e 0 to 0 dl 1787827185 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787827175.815419:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab7800 lustre-OST0001_UUID: changing import state from CONNECTING to IDLE 00000100:00100000:2.0:1787827175.815422:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8de00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572441216:192.168.203.129@tcp:9:kworker.0 00010000:00080000:2.0:1787827175.815434:0:2393:0:(ldlm_request.c:1671:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=46162) req@ffff8cb998a8c3c0 x1874672572441088/t0(0) o9->lustre-OST0000-osc-ffff8cb98543f000@192.168.203.129@tcp:28/4 lens 224/224 e 0 to 0 dl 1787827185 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787827175.815443:0:2393:0:(import.c:2011:ptlrpc_disconnect_idle_interpret()) @@@ inflight=1, refcount=3: rc = 0 req@ffff8cb998a8c3c0 x1874672572441088/t0(0) o9->lustre-OST0000-osc-ffff8cb98543f000@192.168.203.129@tcp:28/4 lens 224/224 e 0 to 0 dl 1787827185 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787827175.815447:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to IDLE 00000100:00100000:2.0:1787827175.815449:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8c3c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572441088:192.168.203.129@tcp:9:kworker.0 00010000:00010000:3.0:1787827176.642370:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787827177.666422:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787827178.690457:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787827179.714344:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787827180.738402:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787827180.930424:0:2396:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8de00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2396:1874672572441344:192.168.203.129@tcp:400:kworker.0 00000100:00100000:3.0:1787827180.930425:0:2397:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8da40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2397:1874672572441472:192.168.203.129@tcp:400:kworker.0 00000100:00100000:2.0:1787827180.930514:0:2396:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8d2c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2396:1874672572441600:192.168.203.129@tcp:400:kworker.0 00010000:00010000:0.0:1787827181.762447:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787827182.786356:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787827183.810505:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787827184.834472:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787827185.858895:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787827186.050503:0:2396:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8c3c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2396:1874672572441856:192.168.203.129@tcp:400:kworker.0 00000100:00100000:3.0:1787827186.050603:0:2397:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8fc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2397:1874672572441728:192.168.203.129@tcp:400:kworker.0 00000100:00100000:3.0:1787827186.050686:0:2397:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8e1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2397:1874672572441984:192.168.203.129@tcp:400:kworker.0 00010000:00010000:0.0:1787827186.882494:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787827187.907594:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787827188.930488:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787827189.954429:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787827190.978417:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787827191.170430:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c6b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2394:1874672572442112:192.168.203.129@tcp:400:kworker.0 00000100:00100000:0.0:1787827191.170444:0:2395:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c68000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2395:1874672572442240:192.168.203.129@tcp:400:kworker.0 00000100:00100000:0.0:1787827191.170534:0:2395:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:0.0:1787827191.170544:0:2395:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c6a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2395:1874672572442368:192.168.203.129@tcp:400:kworker.0 00010000:00010000:3.0:1787827192.002370:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787827193.026381:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787827194.050451:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787827195.074369:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787827196.098375:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000400:2.0:1787827196.290356:0:2397:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1787827180/real 1787827180] req@ffff8cb998a8da40 x1874672572441472/t0(0) o400->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 224/224 e 0 to 1 dl 1787827196 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000400:3.0:1787827196.290356:0:2396:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1787827180/real 1787827180] req@ffff8cb998a8de00 x1874672572441344/t0(0) o400->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 224/224 e 0 to 1 dl 1787827196 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787827196.290406:0:2397:0:(client.c:2547:ptlrpc_expire_one_request()) @@@ TIMEOUT-NORESEND: req@ffff8cb998a8da40 x1874672572441472/t0(0) o400->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 224/224 e 0 to 1 dl 1787827196 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787827196.290419:0:2397:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from FULL to DISCONN 00000100:02000400:2.0:1787827196.290425:0:2397:0:(import.c:182:ptlrpc_set_import_discon()) lustre-MDT0000-mdc-ffff8cb98543f000: Connection to lustre-MDT0000 (at 192.168.203.129@tcp) was lost; in progress operations using this service will wait for recovery to complete 00000100:00100000:1.0:1787827196.290471:0:2395:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c6a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2395:1874672572442496:192.168.203.129@tcp:400:kworker.0 00000100:00100000:1.0:1787827196.290567:0:2395:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c69680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2395:1874672572442752:192.168.203.129@tcp:400:kworker.0 00000100:00080000:2.0:1787827196.304182:0:2397:0:(import.c:418:ptlrpc_pinger_force()) lustre-MDT0000_UUID: waking up pinger s:DISCONN 00000100:00100000:0.0:1787827196.304198:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c692c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:8cef8de3-2326-42de-9d41-f4cb057ed6e8:0:1874672572442624:192.168.203.129@tcp:400:kworker.0 00000100:00100000:2.0:1787827196.304225:0:2397:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8da40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2397:1874672572441472:192.168.203.129@tcp:400:kworker.0 00000100:00080000:0.0:1787827196.304257:0:480:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787827196.304286:0:480:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787827196.304290:0:480:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787827196.304297:0:480:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 106 00000100:00080000:0.0:1787827196.304312:0:480:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:0.0:1787827196.304323:0:480:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787827196.304348:0:480:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c692c0 x1874672572442880/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787827196.304394:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c692c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572442880:192.168.203.129@tcp:38:kworker.0 00000100:00100000:3.0:1787827196.306496:0:2396:0:(client.c:2547:ptlrpc_expire_one_request()) @@@ TIMEOUT-NORESEND: req@ffff8cb998a8de00 x1874672572441344/t0(0) o400->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 224/224 e 0 to 1 dl 1787827196 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827196.306508:0:2396:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from FULL to DISCONN 00000100:02020000:3.0:1787827196.306512:0:2396:0:(import.c:186:ptlrpc_set_import_discon()) MGC192.168.203.129@tcp: Connection to MGS (at 192.168.203.129@tcp) was lost; in progress operations using this service will fail 00000100:00080000:3.0:1787827196.309456:0:2396:0:(import.c:192:ptlrpc_set_import_discon()) import MGS@192.168.203.129@tcp for MGC192.168.203.129@tcp not replayable, auto-deactivating 00000100:00080000:3.0:1787827196.309459:0:2396:0:(import.c:138:ptlrpc_deactivate_import_nolock()) setting import MGS INVALID 00000100:00100000:3.0:1787827196.309464:0:2396:0:(client.c:3589:ptlrpc_abort_inflight()) @@@ inflight req@ffff8cb998a8de00 x1874672572441344/t0(0) o400->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 224/224 e 0 to 1 dl 1787827196 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827196.309473:0:2396:0:(client.c:3589:ptlrpc_abort_inflight()) @@@ inflight req@ffff8cb998a8fc00 x1874672572441728/t0(0) o400->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 224/224 e 0 to 0 dl 1787827202 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827196.309485:0:2396:0:(client.c:3589:ptlrpc_abort_inflight()) @@@ inflight req@ffff8cb999c6b480 x1874672572442112/t0(0) o400->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 224/224 e 0 to 0 dl 1787827207 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827196.309497:0:2396:0:(client.c:3589:ptlrpc_abort_inflight()) @@@ inflight req@ffff8cb999c6a580 x1874672572442496/t0(0) o400->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 224/224 e 0 to 0 dl 1787827212 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 10000000:01000000:3.0:1787827196.309515:0:2396:0:(mgc_request.c:1168:mgc_import_event()) import event 0x808001 00000100:00080000:3.0:1787827196.309519:0:2396:0:(pinger.c:193:ptlrpc_pinger_ir_down()) IR down 10000000:01000000:3.0:1787827196.309521:0:2396:0:(mgc_request.c:1168:mgc_import_event()) import event 0x808002 00000100:00080000:3.0:1787827196.309523:0:2396:0:(import.c:418:ptlrpc_pinger_force()) MGS: waking up pinger s:DISCONN 00000100:00100000:2.0:1787827196.309543:0:2397:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8fc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2397:1874672572441728:192.168.203.129@tcp:400:kworker.0 00000100:00000400:3.0:1787827196.309552:0:2396:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1787827180/real 1787827180] req@ffff8cb998a8d2c0 x1874672572441600/t0(0) o400->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 224/224 e 0 to 1 dl 1787827196 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827196.309553:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c6b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2394:1874672572442112:192.168.203.129@tcp:400:kworker.0 00000100:00100000:1.0:1787827196.309572:0:2395:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c6a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2395:1874672572442496:192.168.203.129@tcp:400:kworker.0 00000100:00100000:3.0:1787827196.309580:0:2396:0:(client.c:2547:ptlrpc_expire_one_request()) @@@ TIMEOUT-NORESEND: req@ffff8cb998a8d2c0 x1874672572441600/t0(0) o400->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 224/224 e 0 to 1 dl 1787827196 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827196.309594:0:2396:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from FULL to DISCONN 00000100:00080000:2.0:1787827196.309595:0:480:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:02000400:3.0:1787827196.309600:0:2396:0:(import.c:182:ptlrpc_set_import_discon()) lustre-MDT0001-mdc-ffff8cb98543f000: Connection to lustre-MDT0001 (at 192.168.203.129@tcp) was lost; in progress operations using this service will wait for recovery to complete 00000100:00080000:2.0:1787827196.309604:0:480:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787827196.309605:0:2396:0:(import.c:418:ptlrpc_pinger_force()) lustre-MDT0001_UUID: waking up pinger s:DISCONN 00000100:00080000:2.0:1787827196.309610:0:480:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827196.309617:0:480:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 106 00000100:00100000:3.0:1787827196.309626:0:2396:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8de00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2396:1874672572441344:192.168.203.129@tcp:400:kworker.0 00000100:00080000:2.0:1787827196.309627:0:480:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:3.0:1787827196.309639:0:2396:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8d2c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2396:1874672572441600:192.168.203.129@tcp:400:kworker.0 00000100:00100000:2.0:1787827196.309650:0:480:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8fc00 x1874672572443008/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787827196.309668:0:480:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827196.309671:0:480:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1787827196.309673:0:480:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827196.309682:0:480:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 106 00000100:00100000:0.0:1787827196.309691:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8fc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572443008:192.168.203.129@tcp:250:kworker.0 00000100:00080000:2.0:1787827196.309691:0:480:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:2.0:1787827196.309701:0:480:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787827196.309712:0:480:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8da40 x1874672572443136/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787827196.309750:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8da40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572443136:192.168.203.129@tcp:38:kworker.0 00010000:00010000:0.0:1787827197.122425:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787827198.146436:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787827199.170473:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787827200.194442:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787827201.218381:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787827201.410514:0:480:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787827201.410524:0:480:0:(pinger.c:247:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:0.0:1787827201.410528:0:480:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787827201.410532:0:480:0:(pinger.c:247:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:0.0:1787827201.410536:0:480:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787827201.410540:0:480:0:(pinger.c:247:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00010000:00010000:0.0:1787827202.242401:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000400:3.0:1787827202.434413:0:2396:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1787827186/real 1787827186] req@ffff8cb998a8c3c0 x1874672572441856/t0(0) o400->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 224/224 e 0 to 1 dl 1787827202 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000400:2.0:1787827202.443242:0:2397:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1787827186/real 1787827186] req@ffff8cb998a8e1c0 x1874672572441984/t0(0) o400->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 224/224 e 0 to 1 dl 1787827202 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787827202.443279:0:2397:0:(client.c:2547:ptlrpc_expire_one_request()) @@@ TIMEOUT-NORESEND: req@ffff8cb998a8e1c0 x1874672572441984/t0(0) o400->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 224/224 e 0 to 1 dl 1787827202 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787827202.443287:0:2397:0:(import.c:214:ptlrpc_set_import_discon()) mdc: import ffff8cb998ab5800 already not connected (conn 2, was 1): CONNECTING 00000100:00100000:2.0:1787827202.443299:0:2397:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8e1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2397:1874672572441984:192.168.203.129@tcp:400:kworker.0 00000100:00100000:3.0:1787827202.445646:0:2396:0:(client.c:2547:ptlrpc_expire_one_request()) @@@ TIMEOUT-NORESEND: req@ffff8cb998a8c3c0 x1874672572441856/t0(0) o400->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 224/224 e 0 to 1 dl 1787827202 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827202.445655:0:2396:0:(import.c:214:ptlrpc_set_import_discon()) mdc: import ffff8cb988ffa000 already not connected (conn 2, was 1): CONNECTING 00000100:00100000:3.0:1787827202.445665:0:2396:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8c3c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2396:1874672572441856:192.168.203.129@tcp:400:kworker.0 00010000:00010000:0.0:1787827203.266431:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787827204.290359:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787827205.314407:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787827206.338403:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787827206.530359:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787827206.530369:0:37:0:(pinger.c:247:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:0.0:1787827206.530373:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787827206.530377:0:37:0:(pinger.c:247:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00100000:2.0:1787827206.530377:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1787827196/real 1787827196] req@ffff8cb998a8da40 x1874672572443136/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827206 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827206.530388:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787827206.530391:0:37:0:(pinger.c:247:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00100000:2.0:1787827206.530406:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8da40 x1874672572443136/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827206 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787827206.530418:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1787827196/real 1787827196] req@ffff8cb998a8fc00 x1874672572443008/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827206 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787827206.530432:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8fc00 x1874672572443008/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827206 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787827206.530443:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1787827196/real 1787827196] req@ffff8cb999c692c0 x1874672572442880/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827206 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787827206.530452:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c692c0 x1874672572442880/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827206 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787827206.530461:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787827206.530474:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787827206.530477:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8da40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572443136:192.168.203.129@tcp:38:kworker.0 00000100:00080000:2.0:1787827206.530481:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787827206.530485:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787827206.530488:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8fc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572443008:192.168.203.129@tcp:250:kworker.0 00000100:00080000:2.0:1787827206.530494:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827206.530495:0:480:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827206.530501:0:480:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787827206.530503:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00080000:3.0:1787827206.530505:0:480:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1787827206.530511:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c692c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572442880:192.168.203.129@tcp:38:kworker.0 00000100:00080000:3.0:1787827206.530515:0:480:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 153 00000100:00080000:3.0:1787827206.530526:0:480:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 5s 00000100:00080000:3.0:1787827206.530535:0:480:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:3.0:1787827206.530562:0:480:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb992314f00 x1874672572443264/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787827206.530591:0:480:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827206.530594:0:480:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787827206.530595:0:480:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827206.530598:0:480:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 153 00000100:00080000:3.0:1787827206.530603:0:480:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 5s 00000100:00080000:3.0:1787827206.530616:0:480:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:2.0:1787827206.530619:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb992314f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572443264:192.168.203.129@tcp:250:kworker.0 00000008:00100000:3.0:1787827206.530636:0:480:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787827206.530657:0:480:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb9923161c0 x1874672572443392/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787827206.530667:0:480:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827206.530670:0:480:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1787827206.530672:0:480:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827206.530675:0:480:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 153 00000100:00080000:3.0:1787827206.530678:0:480:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 5s 00000100:00080000:3.0:1787827206.530681:0:480:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:3.0:1787827206.530685:0:480:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787827206.530696:0:480:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb992315e00 x1874672572443520/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787827206.530710:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb9923161c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572443392:192.168.203.129@tcp:38:kworker.0 00000100:00100000:2.0:1787827206.530743:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb992315e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572443520:192.168.203.129@tcp:38:kworker.0 00010000:00010000:3.0:1787827207.362429:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000400:1.0:1787827207.554463:0:2395:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1787827191/real 1787827191] req@ffff8cb999c6a1c0 x1874672572442368/t0(0) o400->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 224/224 e 0 to 1 dl 1787827207 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787827207.562992:0:2395:0:(client.c:2547:ptlrpc_expire_one_request()) @@@ TIMEOUT-NORESEND: req@ffff8cb999c6a1c0 x1874672572442368/t0(0) o400->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 224/224 e 0 to 1 dl 1787827207 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787827207.563003:0:2395:0:(import.c:214:ptlrpc_set_import_discon()) mdc: import ffff8cb998ab5800 already not connected (conn 3, was 1): CONNECTING 00000100:00000400:1.0:1787827207.563024:0:2395:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1787827191/real 1787827191] req@ffff8cb999c68000 x1874672572442240/t0(0) o400->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 224/224 e 0 to 1 dl 1787827207 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787827207.563034:0:2395:0:(client.c:2547:ptlrpc_expire_one_request()) @@@ TIMEOUT-NORESEND: req@ffff8cb999c68000 x1874672572442240/t0(0) o400->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 224/224 e 0 to 1 dl 1787827207 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787827207.563039:0:2395:0:(import.c:214:ptlrpc_set_import_discon()) mdc: import ffff8cb988ffa000 already not connected (conn 3, was 1): CONNECTING 00000100:00100000:1.0:1787827207.563051:0:2395:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c6a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2395:1874672572442368:192.168.203.129@tcp:400:kworker.0 00000100:00100000:1.0:1787827207.563056:0:2395:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c68000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2395:1874672572442240:192.168.203.129@tcp:400:kworker.0 00010000:00010000:0.0:1787827208.386460:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787827209.410431:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787827210.434384:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000100:3.0:1787827210.562373:0:2385:0:(socklnd_cb.c:2254:ksocknal_find_timed_out_conn()) A connection with 12345-192.168.203.129@tcp (192.168.203.129:988) timed out; the network or node may be down. 00010000:00010000:0.0:1787827211.458464:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787827211.650393:0:480:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827211.650422:0:480:0:(pinger.c:247:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:3.0:1787827211.650426:0:480:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827211.650429:0:480:0:(pinger.c:247:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:3.0:1787827211.650431:0:480:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827211.650434:0:480:0:(pinger.c:247:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00010000:00010000:0.0:1787827212.482395:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000400:1.0:1787827212.674354:0:2395:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1787827196/real 1787827196] req@ffff8cb999c69680 x1874672572442752/t0(0) o400->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 224/224 e 0 to 1 dl 1787827212 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787827212.685207:0:2395:0:(client.c:2547:ptlrpc_expire_one_request()) @@@ TIMEOUT-NORESEND: req@ffff8cb999c69680 x1874672572442752/t0(0) o400->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 224/224 e 0 to 1 dl 1787827212 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787827212.685218:0:2395:0:(import.c:214:ptlrpc_set_import_discon()) mdc: import ffff8cb998ab5800 already not connected (conn 3, was 1): CONNECTING 00000100:00100000:1.0:1787827212.685232:0:2395:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c69680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2395:1874672572442752:192.168.203.129@tcp:400:kworker.0 00010000:00010000:3.0:1787827213.506388:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787827214.530383:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787827215.554414:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787827216.578351:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787827216.770350:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827216.770358:0:37:0:(pinger.c:247:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:2.0:1787827216.770361:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827216.770364:0:37:0:(pinger.c:247:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:2.0:1787827216.770367:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827216.770369:0:37:0:(pinger.c:247:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00010000:00010000:0.0:1787827217.602417:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787827218.626496:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787827219.650373:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787827220.674474:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787827221.698450:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827221.890359:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1787827206/real 1787827206] req@ffff8cb9923161c0 x1874672572443392/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827221 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787827221.890366:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787827221.890374:0:37:0:(pinger.c:247:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:1.0:1787827221.890378:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787827221.890381:0:37:0:(pinger.c:247:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:1.0:1787827221.890384:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787827221.890387:0:37:0:(pinger.c:247:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00100000:3.0:1787827221.890399:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb9923161c0 x1874672572443392/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827221 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827221.890411:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1787827206/real 1787827206] req@ffff8cb992315e00 x1874672572443520/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827221 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827221.890424:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb992315e00 x1874672572443520/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827221 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827221.890435:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1787827206/real 1787827206] req@ffff8cb992314f00 x1874672572443264/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827221 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827221.890444:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb992314f00 x1874672572443264/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827221 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827221.890462:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827221.890477:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00080000:1.0:1787827221.890498:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1787827221.890501:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb9923161c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572443392:192.168.203.129@tcp:38:kworker.0 00000100:00080000:1.0:1787827221.890507:0:37:0:(pinger.c:247:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:3.0:1787827221.890509:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787827221.890510:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787827221.890514:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787827221.890517:0:37:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787827221.890523:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 163 00000100:00080000:3.0:1787827221.890524:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:3.0:1787827221.890528:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb992315e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572443520:192.168.203.129@tcp:38:kworker.0 00000100:00080000:1.0:1787827221.890537:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 10s 00000100:00080000:3.0:1787827221.890537:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827221.890543:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00080000:1.0:1787827221.890546:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:3.0:1787827221.890548:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb992314f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572443264:192.168.203.129@tcp:250:kworker.0 00000008:00100000:1.0:1787827221.890561:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787827221.890597:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f38f00 x1874672572443648/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787827221.890621:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787827221.890625:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1787827221.890628:0:37:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787827221.890633:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 163 00000100:00080000:1.0:1787827221.890637:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 10s 00000100:00080000:1.0:1787827221.890641:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:3.0:1787827221.890645:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f38f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572443648:192.168.203.129@tcp:38:kworker.0 00000008:00100000:1.0:1787827221.890646:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787827221.890661:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f39a40 x1874672572443776/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787827221.890674:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787827221.890677:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787827221.890679:0:37:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787827221.890682:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 163 00000100:00080000:1.0:1787827221.890692:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 10s 00000100:00080000:1.0:1787827221.890704:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:1.0:1787827221.890721:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f3b840 x1874672572443904/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1787827221.890779:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f39a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572443776:192.168.203.129@tcp:38:kworker.0 00000100:00100000:3.0:1787827221.890817:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f3b840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572443904:192.168.203.129@tcp:250:kworker.0 00010000:00010000:3.0:1787827222.722426:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787827223.746391:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787827224.770517:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787827225.794360:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787827226.818466:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787827227.010388:0:480:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827227.010397:0:480:0:(pinger.c:247:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:2.0:1787827227.010401:0:480:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827227.010405:0:480:0:(pinger.c:247:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:2.0:1787827227.010408:0:480:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827227.010411:0:480:0:(pinger.c:247:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00010000:00010000:0.0:1787827227.842416:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787827228.866423:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787827229.890377:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787827230.914825:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787827231.938516:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:0.0:1787827232.066576:0:2382:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1021 -> 192.168.203.129:988 00000400:00000100:0.0:1787827232.066643:0:2382:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.129@tcp at host 192.168.203.129 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:0.0:1787827232.066659:0:2382:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.129@tcp 00000800:00000100:0.0:1787827232.066701:0:2382:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.129@tcp 00000800:00000100:0.0:1787827232.066709:0:2382:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.129@tcp 00000100:00100000:3.0:1787827232.066746:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827221/real 1787827232] req@ffff8cb999f39a40 x1874672572443776/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827241 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827232.066769:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f39a40 x1874672572443776/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827241 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827232.066779:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827232.066795:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:3.0:1787827232.066798:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f39a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572443776:192.168.203.129@tcp:38:kworker.0 00000100:00100000:3.0:1787827232.066804:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827221/real 1787827232] req@ffff8cb999f3b840 x1874672572443904/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827241 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827232.066814:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787827232.066819:0:37:0:(pinger.c:247:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:0.0:1787827232.066822:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787827232.066830:0:37:0:(pinger.c:247:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00100000:3.0:1787827232.066841:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f3b840 x1874672572443904/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827241 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827232.066842:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787827232.066850:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1787827232.066855:0:37:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827232.066858:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827232.066863:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 179 00000100:00080000:0.0:1787827232.066871:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 15s 00000100:00080000:3.0:1787827232.066871:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:3.0:1787827232.066877:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f3b840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572443904:192.168.203.129@tcp:250:kworker.0 00000100:00080000:0.0:1787827232.066882:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:3.0:1787827232.066884:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827221/real 1787827232] req@ffff8cb999f38f00 x1874672572443648/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827241 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:0.0:1787827232.066908:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787827232.066909:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f38f00 x1874672572443648/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827241 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827232.066920:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827232.066929:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:3.0:1787827232.066933:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f38f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572443648:192.168.203.129@tcp:38:kworker.0 00000100:00100000:0.0:1787827232.066943:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c6b480 x1874672572444032/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787827232.066963:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787827232.066966:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787827232.066968:0:37:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787827232.066971:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 179 00000100:00100000:3.0:1787827232.066975:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c6b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572444032:192.168.203.129@tcp:38:kworker.0 00000100:00080000:0.0:1787827232.066977:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 15s 00000100:00080000:0.0:1787827232.066983:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:0.0:1787827232.066993:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c6bc00 x1874672572444160/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787827232.067000:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787827232.067002:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787827232.067004:0:37:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787827232.067006:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 179 00000100:00080000:0.0:1787827232.067008:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 15s 00000100:00080000:0.0:1787827232.067010:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:0.0:1787827232.067021:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787827232.067030:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c68f00 x1874672572444288/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1787827232.067039:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:3.0:1787827232.067068:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827232/real 1787827232] req@ffff8cb999c6b480 x1874672572444032/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827257 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827232.067077:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c6b480 x1874672572444032/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827257 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827232.067082:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827232.067084:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:3.0:1787827232.067092:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c6b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572444032:192.168.203.129@tcp:38:kworker.0 00000100:00100000:3.0:1787827232.067103:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c6bc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572444160:192.168.203.129@tcp:250:kworker.0 00000800:00000100:3.0:1787827232.067118:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:3.0:1787827232.067128:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827232/real 1787827232] req@ffff8cb999c6bc00 x1874672572444160/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827257 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827232.067136:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c6bc00 x1874672572444160/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827257 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827232.067141:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827232.067144:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:3.0:1787827232.067146:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c6bc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572444160:192.168.203.129@tcp:250:kworker.0 00000100:00100000:3.0:1787827232.067149:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c68f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572444288:192.168.203.129@tcp:38:kworker.0 00000800:00000100:3.0:1787827232.067160:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:3.0:1787827232.067166:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827232/real 1787827232] req@ffff8cb999c68f00 x1874672572444288/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827257 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827232.067173:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c68f00 x1874672572444288/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827257 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827232.067178:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827232.067180:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:3.0:1787827232.067182:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c68f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572444288:192.168.203.129@tcp:38:kworker.0 00010000:00010000:0.0:1787827232.962936:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787827233.986406:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787827235.010417:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787827236.034364:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787827237.058399:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787827237.250360:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787827237.250370:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787827237.250374:0:37:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787827237.250385:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 189 00000100:00080000:0.0:1787827237.250394:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787827237.250401:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:0.0:1787827237.250435:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c69e00 x1874672572444416/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787827237.250465:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787827237.250469:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787827237.250471:0:37:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787827237.250475:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 189 00000100:00080000:0.0:1787827237.250479:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787827237.250486:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:0.0:1787827237.250496:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787827237.250511:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c69a40 x1874672572444544/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787827237.250525:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1787827237.250534:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c69e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572444416:192.168.203.129@tcp:250:kworker.0 00000100:00080000:0.0:1787827237.250543:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1787827237.250546:0:37:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787827237.250555:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 189 00000100:00080000:0.0:1787827237.250560:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787827237.250565:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:0.0:1787827237.250571:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787827237.250586:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c6b840 x1874672572444672/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1787827237.250660:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c69a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572444544:192.168.203.129@tcp:38:kworker.0 00000100:00100000:3.0:1787827237.250693:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c6b840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572444672:192.168.203.129@tcp:38:kworker.0 00010000:00010000:3.0:1787827238.082361:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787827239.106405:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787827240.130494:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:1.0:1787827240.322521:0:2383:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1021 -> 192.168.203.129:988 00000400:00000100:1.0:1787827240.322558:0:2383:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.129@tcp at host 192.168.203.129 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:1.0:1787827240.322575:0:2383:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.129@tcp 00000800:00000100:1.0:1787827240.322613:0:2383:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.129@tcp 00000800:00000100:1.0:1787827240.322625:0:2383:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.129@tcp 00000100:00100000:3.0:1787827240.322674:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827237/real 1787827240] req@ffff8cb999c69a40 x1874672572444544/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827262 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827240.322712:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c69a40 x1874672572444544/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827262 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827240.322728:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827240.322735:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:3.0:1787827240.322750:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c69a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572444544:192.168.203.129@tcp:38:kworker.0 00000100:00100000:3.0:1787827240.322760:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827237/real 1787827240] req@ffff8cb999c6b840 x1874672572444672/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827262 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827240.322784:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c6b840 x1874672572444672/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827262 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827240.322794:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827240.322799:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:3.0:1787827240.322803:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c6b840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572444672:192.168.203.129@tcp:38:kworker.0 00000100:00100000:3.0:1787827240.322810:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827237/real 1787827240] req@ffff8cb999c69e00 x1874672572444416/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827262 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827240.322824:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c69e00 x1874672572444416/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827262 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827240.322870:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827240.322874:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:3.0:1787827240.322879:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c69e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572444416:192.168.203.129@tcp:250:kworker.0 00010000:00010000:3.0:1787827241.154391:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787827242.178415:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787827242.370408:0:480:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787827242.370416:0:480:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787827242.370419:0:480:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787827242.370426:0:480:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 194 00000100:00080000:1.0:1787827242.370435:0:480:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787827242.370440:0:480:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:1.0:1787827242.370467:0:480:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f3b0c0 x1874672572444800/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787827242.370488:0:480:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787827242.370491:0:480:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787827242.370492:0:480:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787827242.370494:0:480:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 194 00000100:00080000:1.0:1787827242.370496:0:480:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787827242.370499:0:480:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:1.0:1787827242.370506:0:480:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787827242.370527:0:480:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f39e00 x1874672572444928/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787827242.370536:0:480:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787827242.370538:0:480:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1787827242.370540:0:480:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1787827242.370546:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f3b0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572444800:192.168.203.129@tcp:250:kworker.0 00000100:00080000:1.0:1787827242.370547:0:480:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 194 00000100:00080000:1.0:1787827242.370555:0:480:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787827242.370566:0:480:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:1.0:1787827242.370573:0:480:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787827242.370587:0:480:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f38b40 x1874672572445056/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1787827242.370668:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f39e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572444928:192.168.203.129@tcp:38:kworker.0 00000100:00100000:3.0:1787827242.370699:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f38b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572445056:192.168.203.129@tcp:38:kworker.0 00010000:00010000:0.0:1787827243.202400:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:0.0:1787827243.394681:0:2384:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1021 -> 192.168.203.129:988 00000400:00000100:0.0:1787827243.394732:0:2384:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.129@tcp at host 192.168.203.129 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:0.0:1787827243.394754:0:2384:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.129@tcp 00000800:00000100:0.0:1787827243.394803:0:2384:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.129@tcp 00000800:00000100:0.0:1787827243.394813:0:2384:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.129@tcp 00000100:00100000:3.0:1787827243.394863:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827242/real 1787827243] req@ffff8cb999f39e00 x1874672572444928/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827267 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827243.394889:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f39e00 x1874672572444928/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827267 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827243.394902:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827243.394907:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:3.0:1787827243.394911:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f39e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572444928:192.168.203.129@tcp:38:kworker.0 00000100:00100000:3.0:1787827243.394917:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827242/real 1787827243] req@ffff8cb999f38b40 x1874672572445056/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827267 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827243.394925:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f38b40 x1874672572445056/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827267 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827243.394930:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827243.394933:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:3.0:1787827243.394960:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f38b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572445056:192.168.203.129@tcp:38:kworker.0 00000100:00100000:3.0:1787827243.394969:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827242/real 1787827243] req@ffff8cb999f3b0c0 x1874672572444800/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827267 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827243.394985:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f3b0c0 x1874672572444800/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827267 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827243.394992:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827243.394996:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:3.0:1787827243.394999:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f3b0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572444800:192.168.203.129@tcp:250:kworker.0 00010000:00010000:3.0:1787827244.226480:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787827245.250931:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787827246.274541:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787827247.298653:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787827247.490397:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787827247.490408:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787827247.490412:0:37:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787827247.490423:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 199 00000100:00080000:0.0:1787827247.490435:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787827247.490450:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:0.0:1787827247.490500:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c69e00 x1874672572445184/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787827247.490555:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787827247.490558:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787827247.490559:0:37:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787827247.490563:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 199 00000100:00080000:0.0:1787827247.490567:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787827247.490573:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:0.0:1787827247.490586:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787827247.490600:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c69e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572445184:192.168.203.129@tcp:250:kworker.0 00000100:00100000:0.0:1787827247.490605:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c6b840 x1874672572445312/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787827247.490620:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787827247.490623:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1787827247.490625:0:37:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787827247.490628:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 199 00000100:00080000:0.0:1787827247.490631:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787827247.490635:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:0.0:1787827247.490639:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787827247.490650:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c69a40 x1874672572445440/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1787827247.490768:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c6b840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572445312:192.168.203.129@tcp:38:kworker.0 00000100:00100000:3.0:1787827247.490826:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c69a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572445440:192.168.203.129@tcp:38:kworker.0 00010000:00010000:2.0:1787827248.322588:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787827249.346518:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787827250.370843:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:2.0:1787827250.562476:0:2381:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1021 -> 192.168.203.129:988 00000400:00000100:2.0:1787827250.562515:0:2381:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.129@tcp at host 192.168.203.129 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:2.0:1787827250.562533:0:2381:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.129@tcp 00000800:00000100:2.0:1787827250.562593:0:2381:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.129@tcp 00000800:00000100:2.0:1787827250.562608:0:2381:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.129@tcp 00000100:00100000:3.0:1787827250.562616:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827247/real 1787827250] req@ffff8cb999c6b840 x1874672572445312/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827272 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827250.562641:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c6b840 x1874672572445312/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827272 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827250.562652:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827250.562657:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:3.0:1787827250.562661:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c6b840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572445312:192.168.203.129@tcp:38:kworker.0 00000100:00100000:3.0:1787827250.562679:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827247/real 1787827250] req@ffff8cb999c69a40 x1874672572445440/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827272 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827250.562691:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c69a40 x1874672572445440/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827272 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827250.562698:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827250.562701:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:3.0:1787827250.562704:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c69a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572445440:192.168.203.129@tcp:38:kworker.0 00000100:00100000:3.0:1787827250.562711:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827247/real 1787827250] req@ffff8cb999c69e00 x1874672572445184/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827272 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827250.562721:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c69e00 x1874672572445184/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827272 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827250.562728:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827250.562731:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:3.0:1787827250.562733:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c69e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572445184:192.168.203.129@tcp:250:kworker.0 00010000:00010000:2.0:1787827251.394626:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787827252.418520:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787827252.610392:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827252.610404:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787827252.610409:0:37:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827252.610421:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 204 00000100:00080000:3.0:1787827252.610435:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827252.610442:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:3.0:1787827252.610483:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb992314f00 x1874672572445568/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787827252.610546:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827252.610551:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787827252.610553:0:37:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827252.610558:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 204 00000100:00080000:3.0:1787827252.610563:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827252.610569:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:3.0:1787827252.610581:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787827252.610595:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb992315e00 x1874672572445696/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787827252.610601:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb992314f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572445568:192.168.203.129@tcp:250:kworker.0 00000100:00080000:3.0:1787827252.610613:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827252.610618:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1787827252.610621:0:37:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827252.610625:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 204 00000100:00080000:3.0:1787827252.610630:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827252.610635:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:3.0:1787827252.610640:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787827252.610654:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb9923161c0 x1874672572445824/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1787827252.610701:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787827252.610793:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827252/real 1787827252] req@ffff8cb992314f00 x1874672572445568/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827277 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827252.610827:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb992314f00 x1874672572445568/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827277 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827252.610840:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827252.610844:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787827252.610849:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb992314f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572445568:192.168.203.129@tcp:250:kworker.0 00000100:00100000:0.0:1787827252.610890:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb992315e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572445696:192.168.203.129@tcp:38:kworker.0 00000800:00000100:0.0:1787827252.610919:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787827252.610935:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827252/real 1787827252] req@ffff8cb992315e00 x1874672572445696/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827277 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827252.610948:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb992315e00 x1874672572445696/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827277 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827252.610956:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827252.610960:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787827252.610964:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb992315e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572445696:192.168.203.129@tcp:38:kworker.0 00000100:00100000:0.0:1787827252.610971:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb9923161c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572445824:192.168.203.129@tcp:38:kworker.0 00000800:00000100:0.0:1787827252.611014:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787827252.611029:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827252/real 1787827252] req@ffff8cb9923161c0 x1874672572445824/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827277 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827252.611042:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb9923161c0 x1874672572445824/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827277 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827252.611066:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827252.611071:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787827252.611075:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb9923161c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572445824:192.168.203.129@tcp:38:kworker.0 00010000:00010000:3.0:1787827253.442451:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787827254.466758:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787827255.490972:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787827256.514571:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787827257.538482:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787827257.730318:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827257.730326:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787827257.730329:0:37:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827257.730349:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 209 00000100:00080000:3.0:1787827257.730358:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827257.730364:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:3.0:1787827257.730390:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb992315a40 x1874672572445952/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787827257.730411:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827257.730413:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787827257.730415:0:37:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827257.730417:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 209 00000100:00080000:3.0:1787827257.730420:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827257.730423:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:3.0:1787827257.730430:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787827257.730439:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb992315a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572445952:192.168.203.129@tcp:250:kworker.0 00000100:00100000:3.0:1787827257.730441:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb9923170c0 x1874672572446080/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787827257.730453:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827257.730455:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1787827257.730457:0:37:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827257.730459:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 209 00000100:00080000:3.0:1787827257.730461:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827257.730464:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:3.0:1787827257.730467:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787827257.730476:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb992316940 x1874672572446208/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787827257.730544:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb9923170c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572446080:192.168.203.129@tcp:38:kworker.0 00000100:00100000:0.0:1787827257.730568:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb992316940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572446208:192.168.203.129@tcp:38:kworker.0 00010000:00010000:1.0:1787827258.562647:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787827259.586457:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787827260.610425:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:0.0:1787827260.802724:0:2382:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1021 -> 192.168.203.129:988 00000400:00000100:0.0:1787827260.802786:0:2382:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.129@tcp at host 192.168.203.129 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:0.0:1787827260.802808:0:2382:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.129@tcp 00000800:00000100:0.0:1787827260.802868:0:2382:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.129@tcp 00000800:00000100:0.0:1787827260.802880:0:2382:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.129@tcp 00000100:00100000:3.0:1787827260.803020:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827257/real 1787827260] req@ffff8cb9923170c0 x1874672572446080/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827282 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827260.803066:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb9923170c0 x1874672572446080/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827282 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827260.803078:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827260.803084:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:3.0:1787827260.803089:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb9923170c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572446080:192.168.203.129@tcp:38:kworker.0 00000100:00100000:3.0:1787827260.803097:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827257/real 1787827260] req@ffff8cb992316940 x1874672572446208/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827282 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827260.803110:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb992316940 x1874672572446208/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827282 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827260.803117:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827260.803121:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:3.0:1787827260.803124:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb992316940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572446208:192.168.203.129@tcp:38:kworker.0 00000100:00100000:3.0:1787827260.803131:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827257/real 1787827260] req@ffff8cb992315a40 x1874672572445952/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827282 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827260.803144:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb992315a40 x1874672572445952/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827282 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827260.803151:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827260.803155:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:3.0:1787827260.803192:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb992315a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572445952:192.168.203.129@tcp:250:kworker.0 00010000:00010000:1.0:1787827261.634358:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787827262.658387:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787827262.850546:0:480:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827262.850561:0:480:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787827262.850567:0:480:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827262.850582:0:480:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 215 00000100:00080000:2.0:1787827262.850598:0:480:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827262.850608:0:480:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:2.0:1787827262.850662:0:480:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8fc00 x1874672572446336/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787827262.850746:0:480:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827262.850751:0:480:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787827262.850753:0:480:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827262.850758:0:480:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 215 00000100:00080000:2.0:1787827262.850762:0:480:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827262.850767:0:480:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:2.0:1787827262.850780:0:480:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787827262.850813:0:480:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8da40 x1874672572446464/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787827262.850829:0:480:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827262.850834:0:480:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00100000:3.0:1787827262.850846:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8fc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572446336:192.168.203.129@tcp:250:kworker.0 00000100:00080000:2.0:1787827262.850862:0:480:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827262.850871:0:480:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 215 00000100:00080000:2.0:1787827262.850877:0:480:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827262.850884:0:480:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:2.0:1787827262.850892:0:480:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787827262.850927:0:480:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8e1c0 x1874672572446592/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1787827262.850965:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:3.0:1787827262.851031:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827262/real 1787827262] req@ffff8cb998a8fc00 x1874672572446336/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827287 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827262.851055:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8fc00 x1874672572446336/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827287 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827262.851068:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827262.851073:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:3.0:1787827262.851078:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8fc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572446336:192.168.203.129@tcp:250:kworker.0 00000100:00100000:3.0:1787827262.851111:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8da40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572446464:192.168.203.129@tcp:38:kworker.0 00000800:00000100:3.0:1787827262.851139:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:3.0:1787827262.851154:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827262/real 1787827262] req@ffff8cb998a8da40 x1874672572446464/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827287 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827262.851167:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8da40 x1874672572446464/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827287 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827262.851174:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827262.851178:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:3.0:1787827262.851181:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8da40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572446464:192.168.203.129@tcp:38:kworker.0 00000100:00100000:3.0:1787827262.851187:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8e1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572446592:192.168.203.129@tcp:38:kworker.0 00000800:00000100:3.0:1787827262.851209:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:3.0:1787827262.851233:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827262/real 1787827262] req@ffff8cb998a8e1c0 x1874672572446592/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827287 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827262.851366:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8e1c0 x1874672572446592/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827287 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827262.851377:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827262.851380:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:3.0:1787827262.851384:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8e1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572446592:192.168.203.129@tcp:38:kworker.0 00010000:00010000:2.0:1787827263.682417:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787827264.706444:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787827265.731460:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787827266.754856:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787827267.778915:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787827267.970351:0:480:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827267.970365:0:480:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787827267.970368:0:480:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827267.970376:0:480:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 220 00000100:00080000:2.0:1787827267.970385:0:480:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827267.970391:0:480:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:2.0:1787827267.970420:0:480:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8d680 x1874672572446720/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787827267.970445:0:480:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827267.970447:0:480:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787827267.970449:0:480:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827267.970451:0:480:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 220 00000100:00080000:2.0:1787827267.970454:0:480:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827267.970456:0:480:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:2.0:1787827267.970463:0:480:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787827267.970489:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8d680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572446720:192.168.203.129@tcp:250:kworker.0 00000100:00100000:2.0:1787827267.970497:0:480:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8cf00 x1874672572446848/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787827267.970511:0:480:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827267.970515:0:480:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1787827267.970518:0:480:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827267.970522:0:480:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 220 00000100:00080000:2.0:1787827267.970537:0:480:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827267.970541:0:480:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:2.0:1787827267.970546:0:480:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000800:00000100:3.0:1787827267.970549:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787827267.970560:0:480:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8f0c0 x1874672572446976/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1787827267.970598:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827267/real 1787827267] req@ffff8cb998a8d680 x1874672572446720/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827292 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827267.970613:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8d680 x1874672572446720/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827292 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827267.970621:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827267.970625:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:3.0:1787827267.970628:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8d680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572446720:192.168.203.129@tcp:250:kworker.0 00000100:00100000:3.0:1787827267.970662:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8cf00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572446848:192.168.203.129@tcp:38:kworker.0 00000800:00000100:3.0:1787827267.970681:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:3.0:1787827267.970692:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827267/real 1787827267] req@ffff8cb998a8cf00 x1874672572446848/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827292 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827267.970705:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8cf00 x1874672572446848/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827292 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827267.970728:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827267.970733:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:3.0:1787827267.970737:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8cf00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572446848:192.168.203.129@tcp:38:kworker.0 00000100:00100000:3.0:1787827267.970745:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8f0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572446976:192.168.203.129@tcp:38:kworker.0 00000800:00000100:3.0:1787827267.970765:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:3.0:1787827267.970773:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827267/real 1787827267] req@ffff8cb998a8f0c0 x1874672572446976/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827292 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827267.970781:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8f0c0 x1874672572446976/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827292 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827267.970787:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827267.970789:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:3.0:1787827267.970791:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8f0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572446976:192.168.203.129@tcp:38:kworker.0 00010000:00010000:1.0:1787827268.802958:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787827269.826701:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787827270.850444:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787827271.874429:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787827272.898414:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787827273.090385:0:480:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827273.090396:0:480:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787827273.090400:0:480:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827273.090409:0:480:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 225 00000100:00080000:2.0:1787827273.090418:0:480:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827273.090427:0:480:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:2.0:1787827273.090463:0:480:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8e580 x1874672572447104/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787827273.090496:0:480:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827273.090499:0:480:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787827273.090501:0:480:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827273.090517:0:480:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 225 00000100:00080000:2.0:1787827273.090521:0:480:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827273.090525:0:480:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:2.0:1787827273.090535:0:480:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787827273.090548:0:480:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8c780 x1874672572447232/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787827273.090557:0:480:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827273.090560:0:480:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1787827273.090562:0:480:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827273.090565:0:480:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 225 00000100:00080000:2.0:1787827273.090568:0:480:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827273.090571:0:480:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:2.0:1787827273.090578:0:480:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787827273.090590:0:480:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8f480 x1874672572447360/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787827273.090665:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8e580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572447104:192.168.203.129@tcp:250:kworker.0 00000800:00000100:2.0:1787827273.090724:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787827273.090769:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827273/real 1787827273] req@ffff8cb998a8e580 x1874672572447104/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827298 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787827273.090783:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8e580 x1874672572447104/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827298 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787827273.090795:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787827273.090798:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787827273.090806:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8e580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572447104:192.168.203.129@tcp:250:kworker.0 00000100:00100000:2.0:1787827273.090812:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8c780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572447232:192.168.203.129@tcp:38:kworker.0 00000800:00000100:2.0:1787827273.090830:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787827273.090841:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827273/real 1787827273] req@ffff8cb998a8c780 x1874672572447232/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827298 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787827273.090854:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8c780 x1874672572447232/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827298 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787827273.090863:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787827273.090867:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787827273.090871:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8c780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572447232:192.168.203.129@tcp:38:kworker.0 00000100:00100000:2.0:1787827273.090882:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572447360:192.168.203.129@tcp:38:kworker.0 00000800:00000100:2.0:1787827273.090905:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787827273.090919:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827273/real 1787827273] req@ffff8cb998a8f480 x1874672572447360/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827298 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787827273.090932:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8f480 x1874672572447360/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827298 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787827273.090938:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787827273.090941:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787827273.090943:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572447360:192.168.203.129@tcp:38:kworker.0 00010000:00010000:0.0:1787827273.922466:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787827274.946432:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787827275.970461:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787827276.994452:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787827278.018433:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787827278.210422:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827278.210430:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787827278.210433:0:37:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827278.210441:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 230 00000100:00080000:3.0:1787827278.210450:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827278.210456:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:3.0:1787827278.210486:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb992315a40 x1874672572447488/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787827278.210511:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827278.210528:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787827278.210530:0:37:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827278.210534:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 230 00000100:00080000:3.0:1787827278.210537:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827278.210540:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:3.0:1787827278.210548:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787827278.210559:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb992316940 x1874672572447616/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787827278.210562:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb992315a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572447488:192.168.203.129@tcp:250:kworker.0 00000100:00080000:3.0:1787827278.210596:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827278.210599:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1787827278.210600:0:37:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827278.210604:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 230 00000100:00080000:3.0:1787827278.210608:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827278.210611:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:3.0:1787827278.210615:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787827278.210631:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb9923170c0 x1874672572447744/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787827278.210705:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb992316940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572447616:192.168.203.129@tcp:38:kworker.0 00000100:00100000:2.0:1787827278.210729:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb9923170c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572447744:192.168.203.129@tcp:38:kworker.0 00010000:00010000:0.0:1787827279.042408:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787827280.066397:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787827281.090638:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:1.0:1787827281.282529:0:2383:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1021 -> 192.168.203.129:988 00000400:00000100:1.0:1787827281.282588:0:2383:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.129@tcp at host 192.168.203.129 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:1.0:1787827281.282605:0:2383:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.129@tcp 00000800:00000100:1.0:1787827281.282649:0:2383:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.129@tcp 00000800:00000100:1.0:1787827281.282659:0:2383:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.129@tcp 00000100:00100000:2.0:1787827281.282689:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827278/real 1787827281] req@ffff8cb992316940 x1874672572447616/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827303 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787827281.282718:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb992316940 x1874672572447616/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827303 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787827281.282729:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787827281.282735:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787827281.282739:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb992316940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572447616:192.168.203.129@tcp:38:kworker.0 00000100:00100000:2.0:1787827281.282748:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827278/real 1787827281] req@ffff8cb9923170c0 x1874672572447744/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827303 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787827281.282761:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb9923170c0 x1874672572447744/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827303 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787827281.282770:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787827281.282774:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787827281.282776:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb9923170c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572447744:192.168.203.129@tcp:38:kworker.0 00000100:00100000:2.0:1787827281.282782:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827278/real 1787827281] req@ffff8cb992315a40 x1874672572447488/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827303 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787827281.282792:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb992315a40 x1874672572447488/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827303 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787827281.282814:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787827281.282820:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787827281.282825:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb992315a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572447488:192.168.203.129@tcp:250:kworker.0 00010000:00010000:2.0:1787827282.114462:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787827283.138472:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787827283.330341:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787827283.330349:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787827283.330353:0:37:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787827283.330361:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 235 00000100:00080000:0.0:1787827283.330370:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787827283.330387:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:0.0:1787827283.330422:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c68f00 x1874672572447872/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787827283.330451:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787827283.330454:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787827283.330455:0:37:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787827283.330458:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 235 00000100:00080000:0.0:1787827283.330462:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787827283.330466:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:0.0:1787827283.330475:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787827283.330487:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c6bc00 x1874672572448000/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787827283.330493:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c68f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572447872:192.168.203.129@tcp:250:kworker.0 00000100:00080000:0.0:1787827283.330503:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787827283.330507:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1787827283.330508:0:37:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787827283.330512:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 235 00000100:00080000:0.0:1787827283.330514:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787827283.330517:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:0.0:1787827283.330520:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787827283.330531:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c6b480 x1874672572448128/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1787827283.330574:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787827283.330624:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827283/real 1787827283] req@ffff8cb999c68f00 x1874672572447872/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827308 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787827283.330644:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c68f00 x1874672572447872/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827308 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787827283.330670:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787827283.330676:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787827283.330683:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c68f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572447872:192.168.203.129@tcp:250:kworker.0 00000100:00100000:2.0:1787827283.330710:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c6bc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572448000:192.168.203.129@tcp:38:kworker.0 00000800:00000100:2.0:1787827283.330753:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787827283.330770:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827283/real 1787827283] req@ffff8cb999c6bc00 x1874672572448000/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827308 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787827283.330783:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c6bc00 x1874672572448000/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827308 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787827283.330791:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787827283.330800:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787827283.330812:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c6bc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572448000:192.168.203.129@tcp:38:kworker.0 00000100:00100000:2.0:1787827283.330819:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c6b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572448128:192.168.203.129@tcp:38:kworker.0 00000800:00000100:2.0:1787827283.330842:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787827283.330855:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827283/real 1787827283] req@ffff8cb999c6b480 x1874672572448128/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827308 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787827283.330867:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c6b480 x1874672572448128/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827308 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787827283.330875:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787827283.330878:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787827283.330882:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c6b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572448128:192.168.203.129@tcp:38:kworker.0 00010000:00010000:2.0:1787827284.162422:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787827285.186432:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787827286.210678:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787827287.234553:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787827288.258539:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787827288.450343:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827288.450352:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787827288.450356:0:37:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827288.450365:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 240 00000100:00080000:3.0:1787827288.450374:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827288.450381:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:3.0:1787827288.450430:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb992315a40 x1874672572448256/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787827288.450459:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827288.450463:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787827288.450465:0:37:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827288.450469:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 240 00000100:00080000:3.0:1787827288.450475:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827288.450480:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:3.0:1787827288.450492:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787827288.450497:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb992315a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572448256:192.168.203.129@tcp:250:kworker.0 00000100:00100000:3.0:1787827288.450523:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb9923170c0 x1874672572448384/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787827288.450537:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827288.450540:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1787827288.450542:0:37:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827288.450545:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 240 00000100:00080000:3.0:1787827288.450556:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827288.450560:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:3.0:1787827288.450565:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000800:00000100:2.0:1787827288.450571:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:3.0:1787827288.450579:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb992316940 x1874672572448512/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787827288.450634:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827288/real 1787827288] req@ffff8cb992315a40 x1874672572448256/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827313 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787827288.450659:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb992315a40 x1874672572448256/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827313 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787827288.450669:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787827288.450672:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787827288.450675:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb992315a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572448256:192.168.203.129@tcp:250:kworker.0 00000100:00100000:2.0:1787827288.450694:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb9923170c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572448384:192.168.203.129@tcp:38:kworker.0 00000800:00000100:2.0:1787827288.450719:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787827288.450729:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827288/real 1787827288] req@ffff8cb9923170c0 x1874672572448384/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827313 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787827288.450738:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb9923170c0 x1874672572448384/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827313 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787827288.450743:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787827288.450746:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787827288.450748:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb9923170c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572448384:192.168.203.129@tcp:38:kworker.0 00000100:00100000:2.0:1787827288.450752:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb992316940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572448512:192.168.203.129@tcp:38:kworker.0 00000800:00000100:2.0:1787827288.450763:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787827288.450770:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827288/real 1787827288] req@ffff8cb992316940 x1874672572448512/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827313 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787827288.450777:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb992316940 x1874672572448512/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827313 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787827288.450782:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787827288.450784:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787827288.450786:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb992316940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572448512:192.168.203.129@tcp:38:kworker.0 00010000:00010000:1.0:1787827289.282559:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787827290.306485:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787827291.330373:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787827292.354362:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787827293.378400:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787827293.570352:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827293.570360:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787827293.570378:0:37:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827293.570386:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 245 00000100:00080000:3.0:1787827293.570395:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827293.570401:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:3.0:1787827293.570429:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb9923161c0 x1874672572448640/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787827293.570459:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827293.570462:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787827293.570463:0:37:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827293.570466:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 245 00000100:00080000:3.0:1787827293.570479:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827293.570484:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:3.0:1787827293.570493:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787827293.570504:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb992315e00 x1874672572448768/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787827293.570512:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb9923161c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572448640:192.168.203.129@tcp:250:kworker.0 00000100:00080000:3.0:1787827293.570515:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827293.570518:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1787827293.570520:0:37:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827293.570526:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 245 00000100:00080000:3.0:1787827293.570529:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827293.570531:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:3.0:1787827293.570534:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787827293.570541:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb992314f00 x1874672572448896/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1787827293.570589:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787827293.570638:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827293/real 1787827293] req@ffff8cb9923161c0 x1874672572448640/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827318 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787827293.570675:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb9923161c0 x1874672572448640/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827318 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787827293.570683:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787827293.570687:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787827293.570691:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb9923161c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572448640:192.168.203.129@tcp:250:kworker.0 00000100:00100000:2.0:1787827293.570714:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb992315e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572448768:192.168.203.129@tcp:38:kworker.0 00000800:00000100:2.0:1787827293.570734:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787827293.570744:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827293/real 1787827293] req@ffff8cb992315e00 x1874672572448768/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827318 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787827293.570758:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb992315e00 x1874672572448768/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827318 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787827293.570764:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787827293.570767:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787827293.570769:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb992315e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572448768:192.168.203.129@tcp:38:kworker.0 00000100:00100000:2.0:1787827293.570773:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb992314f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572448896:192.168.203.129@tcp:38:kworker.0 00000800:00000100:2.0:1787827293.570788:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787827293.570795:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827293/real 1787827293] req@ffff8cb992314f00 x1874672572448896/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827318 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787827293.570803:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb992314f00 x1874672572448896/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827318 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787827293.570830:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787827293.570833:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787827293.570836:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb992314f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572448896:192.168.203.129@tcp:38:kworker.0 00010000:00010000:3.0:1787827294.402346:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787827295.426351:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787827296.450376:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787827297.474364:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787827298.498465:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787827298.690379:0:480:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827298.690387:0:480:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787827298.690391:0:480:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827298.690400:0:480:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 250 00000100:00080000:3.0:1787827298.690408:0:480:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827298.690414:0:480:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:3.0:1787827298.690471:0:480:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb992317480 x1874672572449024/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787827298.690492:0:480:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827298.690495:0:480:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787827298.690497:0:480:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827298.690499:0:480:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 250 00000100:00080000:3.0:1787827298.690502:0:480:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827298.690505:0:480:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:3.0:1787827298.690524:0:480:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787827298.690532:0:480:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb992316580 x1874672572449152/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787827298.690540:0:480:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827298.690543:0:480:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1787827298.690544:0:480:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827298.690547:0:480:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 250 00000100:00080000:3.0:1787827298.690562:0:480:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827298.690565:0:480:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:3.0:1787827298.690569:0:480:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787827298.690578:0:480:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb992314780 x1874672572449280/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787827298.690591:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb992317480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572449024:192.168.203.129@tcp:250:kworker.0 00000800:00000100:2.0:1787827298.690673:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787827298.690727:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827298/real 1787827298] req@ffff8cb992317480 x1874672572449024/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827323 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787827298.690765:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb992317480 x1874672572449024/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827323 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787827298.690777:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787827298.690780:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787827298.690784:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb992317480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572449024:192.168.203.129@tcp:250:kworker.0 00000100:00100000:2.0:1787827298.690791:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb992316580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572449152:192.168.203.129@tcp:38:kworker.0 00000800:00000100:2.0:1787827298.690811:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787827298.690820:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827298/real 1787827298] req@ffff8cb992316580 x1874672572449152/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827323 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787827298.690829:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb992316580 x1874672572449152/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827323 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787827298.690834:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787827298.690837:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787827298.690852:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb992316580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572449152:192.168.203.129@tcp:38:kworker.0 00000100:00100000:2.0:1787827298.690882:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb992314780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572449280:192.168.203.129@tcp:38:kworker.0 00000800:00000100:2.0:1787827298.690897:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787827298.690904:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827298/real 1787827298] req@ffff8cb992314780 x1874672572449280/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827323 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787827298.690912:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb992314780 x1874672572449280/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827323 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787827298.690917:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787827298.690920:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787827298.690922:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb992314780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572449280:192.168.203.129@tcp:38:kworker.0 00010000:00010000:2.0:1787827299.522414:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787827300.546436:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787827301.570428:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787827302.594515:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787827303.618427:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787827303.810359:0:480:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827303.810371:0:480:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787827303.810376:0:480:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827303.810389:0:480:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 255 00000100:00080000:3.0:1787827303.810414:0:480:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827303.810424:0:480:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:3.0:1787827303.810469:0:480:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb992314b40 x1874672572449408/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787827303.810502:0:480:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827303.810506:0:480:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787827303.810509:0:480:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827303.810512:0:480:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 255 00000100:00080000:3.0:1787827303.810516:0:480:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827303.810520:0:480:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:3.0:1787827303.810532:0:480:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787827303.810547:0:480:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb992314780 x1874672572449536/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787827303.810571:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb992314b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572449408:192.168.203.129@tcp:250:kworker.0 00000100:00080000:3.0:1787827303.810595:0:480:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827303.810600:0:480:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1787827303.810603:0:480:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827303.810610:0:480:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 255 00000100:00080000:3.0:1787827303.810615:0:480:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827303.810619:0:480:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:3.0:1787827303.810626:0:480:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787827303.810641:0:480:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb992316580 x1874672572449664/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1787827303.810646:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787827303.810700:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827303/real 1787827303] req@ffff8cb992314b40 x1874672572449408/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827328 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787827303.810719:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb992314b40 x1874672572449408/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827328 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787827303.810729:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787827303.810732:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787827303.810735:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb992314b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572449408:192.168.203.129@tcp:250:kworker.0 00000100:00100000:2.0:1787827303.810764:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb992314780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572449536:192.168.203.129@tcp:38:kworker.0 00000800:00000100:2.0:1787827303.810795:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787827303.810805:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827303/real 1787827303] req@ffff8cb992314780 x1874672572449536/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827328 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787827303.810813:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb992314780 x1874672572449536/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827328 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787827303.810818:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787827303.810820:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787827303.810823:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb992314780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572449536:192.168.203.129@tcp:38:kworker.0 00000100:00100000:2.0:1787827303.810827:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb992316580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572449664:192.168.203.129@tcp:38:kworker.0 00000800:00000100:2.0:1787827303.810838:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787827303.810850:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827303/real 1787827303] req@ffff8cb992316580 x1874672572449664/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827328 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787827303.810858:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb992316580 x1874672572449664/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827328 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787827303.810864:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787827303.810866:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787827303.810868:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb992316580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572449664:192.168.203.129@tcp:38:kworker.0 00010000:00010000:0.0:1787827304.642427:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787827305.666424:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787827306.690407:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787827307.714428:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787827308.739011:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787827308.930344:0:480:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827308.930352:0:480:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787827308.930356:0:480:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827308.930374:0:480:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 261 00000100:00080000:3.0:1787827308.930384:0:480:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827308.930391:0:480:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:3.0:1787827308.930421:0:480:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb992317480 x1874672572449792/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787827308.930443:0:480:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827308.930446:0:480:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787827308.930448:0:480:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827308.930450:0:480:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 261 00000100:00080000:3.0:1787827308.930452:0:480:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827308.930468:0:480:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:3.0:1787827308.930477:0:480:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787827308.930485:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb992317480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572449792:192.168.203.129@tcp:250:kworker.0 00000100:00100000:3.0:1787827308.930488:0:480:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb992314f00 x1874672572449920/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787827308.930499:0:480:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827308.930502:0:480:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1787827308.930503:0:480:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827308.930506:0:480:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 261 00000100:00080000:3.0:1787827308.930509:0:480:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827308.930511:0:480:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:3.0:1787827308.930515:0:480:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787827308.930524:0:480:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb992315e00 x1874672572450048/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1787827308.930544:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787827308.930580:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827308/real 1787827308] req@ffff8cb992317480 x1874672572449792/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827333 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787827308.930608:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb992317480 x1874672572449792/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827333 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787827308.930617:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787827308.930620:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787827308.930624:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb992317480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572449792:192.168.203.129@tcp:250:kworker.0 00000100:00100000:2.0:1787827308.930645:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb992314f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572449920:192.168.203.129@tcp:38:kworker.0 00000800:00000100:2.0:1787827308.930666:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787827308.930679:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827308/real 1787827308] req@ffff8cb992314f00 x1874672572449920/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827333 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787827308.930690:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb992314f00 x1874672572449920/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827333 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787827308.930696:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787827308.930699:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787827308.930702:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb992314f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572449920:192.168.203.129@tcp:38:kworker.0 00000100:00100000:2.0:1787827308.930707:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb992315e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572450048:192.168.203.129@tcp:38:kworker.0 00000800:00000100:2.0:1787827308.930720:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787827308.930730:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827308/real 1787827308] req@ffff8cb992315e00 x1874672572450048/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827333 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787827308.930738:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb992315e00 x1874672572450048/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827333 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787827308.930753:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787827308.930757:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787827308.930760:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb992315e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572450048:192.168.203.129@tcp:38:kworker.0 00010000:00010000:3.0:1787827309.762477:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787827310.786598:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787827311.810432:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787827312.834394:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787827313.858405:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787827314.050360:0:480:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827314.050369:0:480:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787827314.050372:0:480:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827314.050380:0:480:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 266 00000100:00080000:3.0:1787827314.050389:0:480:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827314.050395:0:480:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:3.0:1787827314.050427:0:480:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb9923161c0 x1874672572450176/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787827314.050450:0:480:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827314.050466:0:480:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787827314.050468:0:480:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827314.050471:0:480:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 266 00000100:00080000:3.0:1787827314.050475:0:480:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827314.050478:0:480:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:3.0:1787827314.050487:0:480:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787827314.050497:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb9923161c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572450176:192.168.203.129@tcp:250:kworker.0 00000100:00100000:3.0:1787827314.050499:0:480:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb992316940 x1874672572450304/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787827314.050511:0:480:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827314.050513:0:480:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1787827314.050515:0:480:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827314.050518:0:480:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 266 00000100:00080000:3.0:1787827314.050520:0:480:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827314.050522:0:480:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:3.0:1787827314.050526:0:480:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787827314.050534:0:480:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb9923170c0 x1874672572450432/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787827314.050590:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb992316940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572450304:192.168.203.129@tcp:38:kworker.0 00000100:00100000:2.0:1787827314.050624:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb9923170c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572450432:192.168.203.129@tcp:38:kworker.0 00010000:00010000:2.0:1787827314.882462:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787827315.906400:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787827316.930558:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:0.0:1787827317.122706:0:2384:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.129:988 00000400:00000100:0.0:1787827317.122764:0:2384:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.129@tcp at host 192.168.203.129 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:0.0:1787827317.122784:0:2384:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.129@tcp 00000800:00000100:0.0:1787827317.122835:0:2384:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.129@tcp 00000800:00000100:0.0:1787827317.122846:0:2384:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.129@tcp 00000100:00100000:2.0:1787827317.122907:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827314/real 1787827317] req@ffff8cb992316940 x1874672572450304/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827339 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787827317.122941:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb992316940 x1874672572450304/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827339 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787827317.122952:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787827317.122957:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787827317.122961:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb992316940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572450304:192.168.203.129@tcp:38:kworker.0 00000100:00100000:2.0:1787827317.122968:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827314/real 1787827317] req@ffff8cb9923170c0 x1874672572450432/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827339 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787827317.122977:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb9923170c0 x1874672572450432/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827339 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787827317.123010:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787827317.123014:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787827317.123018:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb9923170c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572450432:192.168.203.129@tcp:38:kworker.0 00000100:00100000:2.0:1787827317.123026:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827314/real 1787827317] req@ffff8cb9923161c0 x1874672572450176/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827339 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787827317.123041:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb9923161c0 x1874672572450176/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827339 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787827317.123049:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787827317.123053:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787827317.123056:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb9923161c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572450176:192.168.203.129@tcp:250:kworker.0 00010000:00010000:2.0:1787827317.954646:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787827318.978453:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787827319.170391:0:480:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827319.170399:0:480:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787827319.170403:0:480:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827319.170412:0:480:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 271 00000100:00080000:3.0:1787827319.170421:0:480:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827319.170444:0:480:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:3.0:1787827319.170476:0:480:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb992315a40 x1874672572450560/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787827319.170509:0:480:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827319.170512:0:480:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787827319.170514:0:480:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827319.170516:0:480:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 271 00000100:00080000:3.0:1787827319.170519:0:480:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827319.170522:0:480:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:3.0:1787827319.170530:0:480:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787827319.170540:0:480:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb9923161c0 x1874672572450688/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787827319.170549:0:480:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827319.170552:0:480:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1787827319.170554:0:480:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827319.170556:0:480:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 271 00000100:00080000:3.0:1787827319.170559:0:480:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827319.170562:0:480:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:3.0:1787827319.170566:0:480:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787827319.170573:0:480:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb9923170c0 x1874672572450816/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787827319.170587:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb992315a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572450560:192.168.203.129@tcp:250:kworker.0 00000800:00000100:2.0:1787827319.170740:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787827319.170817:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827319/real 1787827319] req@ffff8cb992315a40 x1874672572450560/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827344 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787827319.170842:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb992315a40 x1874672572450560/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827344 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787827319.170858:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787827319.170863:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787827319.170869:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb992315a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572450560:192.168.203.129@tcp:250:kworker.0 00000100:00100000:2.0:1787827319.170880:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb9923161c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572450688:192.168.203.129@tcp:38:kworker.0 00000800:00000100:2.0:1787827319.170933:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787827319.170970:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827319/real 1787827319] req@ffff8cb9923161c0 x1874672572450688/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827344 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787827319.170985:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb9923161c0 x1874672572450688/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827344 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787827319.170993:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787827319.170997:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787827319.171000:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb9923161c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572450688:192.168.203.129@tcp:38:kworker.0 00000100:00100000:2.0:1787827319.171042:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb9923170c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572450816:192.168.203.129@tcp:38:kworker.0 00000800:00000100:2.0:1787827319.171067:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787827319.171080:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827319/real 1787827319] req@ffff8cb9923170c0 x1874672572450816/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827344 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787827319.171091:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb9923170c0 x1874672572450816/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827344 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787827319.171099:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787827319.171103:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787827319.171106:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb9923170c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572450816:192.168.203.129@tcp:38:kworker.0 00010000:00010000:2.0:1787827320.002588:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787827321.026950:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787827322.050489:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787827323.074982:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787827324.098631:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787827324.290346:0:480:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787827324.290360:0:480:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787827324.290365:0:480:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787827324.290378:0:480:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 276 00000100:00080000:0.0:1787827324.290392:0:480:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787827324.290401:0:480:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:0.0:1787827324.290449:0:480:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c69680 x1874672572450944/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787827324.290479:0:480:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787827324.290482:0:480:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787827324.290484:0:480:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787827324.290488:0:480:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 276 00000100:00080000:0.0:1787827324.290493:0:480:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787827324.290498:0:480:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:0.0:1787827324.290511:0:480:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787827324.290528:0:480:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c68000 x1874672572451072/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787827324.290530:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c69680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572450944:192.168.203.129@tcp:250:kworker.0 00000100:00080000:0.0:1787827324.290564:0:480:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787827324.290569:0:480:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1787827324.290572:0:480:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787827324.290577:0:480:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 276 00000100:00080000:0.0:1787827324.290583:0:480:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787827324.290588:0:480:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:0.0:1787827324.290595:0:480:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787827324.290609:0:480:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c6a1c0 x1874672572451200/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1787827324.290655:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787827324.290712:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827324/real 1787827324] req@ffff8cb999c69680 x1874672572450944/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827349 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787827324.290731:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c69680 x1874672572450944/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827349 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787827324.290741:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787827324.290744:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787827324.290747:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c69680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572450944:192.168.203.129@tcp:250:kworker.0 00000100:00100000:2.0:1787827324.290770:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c68000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572451072:192.168.203.129@tcp:38:kworker.0 00000800:00000100:2.0:1787827324.290793:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787827324.290808:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827324/real 1787827324] req@ffff8cb999c68000 x1874672572451072/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827349 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787827324.290822:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c68000 x1874672572451072/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827349 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787827324.290832:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787827324.290836:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787827324.290840:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c68000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572451072:192.168.203.129@tcp:38:kworker.0 00000100:00100000:2.0:1787827324.290845:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c6a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572451200:192.168.203.129@tcp:38:kworker.0 00000800:00000100:2.0:1787827324.290862:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787827324.290870:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827324/real 1787827324] req@ffff8cb999c6a1c0 x1874672572451200/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827349 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787827324.290889:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c6a1c0 x1874672572451200/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827349 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787827324.290895:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787827324.290897:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787827324.290899:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c6a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572451200:192.168.203.129@tcp:38:kworker.0 00010000:00010000:3.0:1787827325.122533:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787827326.146973:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787827327.170488:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787827328.194435:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787827329.218463:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787827329.411311:0:480:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787827329.411322:0:480:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787827329.411326:0:480:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787827329.411337:0:480:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 281 00000100:00080000:0.0:1787827329.411349:0:480:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787827329.411356:0:480:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:0.0:1787827329.411397:0:480:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c692c0 x1874672572451328/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787827329.411422:0:480:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787827329.411425:0:480:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787827329.411426:0:480:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787827329.411429:0:480:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 281 00000100:00080000:0.0:1787827329.411432:0:480:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787827329.411434:0:480:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:0.0:1787827329.411451:0:480:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787827329.411462:0:480:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c6a580 x1874672572451456/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787827329.411470:0:480:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787827329.411473:0:480:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1787827329.411474:0:480:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787827329.411477:0:480:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 281 00000100:00080000:0.0:1787827329.411480:0:480:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787827329.411484:0:480:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:0.0:1787827329.411488:0:480:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787827329.411494:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c692c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572451328:192.168.203.129@tcp:250:kworker.0 00000100:00100000:0.0:1787827329.411498:0:480:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c6a1c0 x1874672572451584/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1787827329.411572:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787827329.411612:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827329/real 1787827329] req@ffff8cb999c692c0 x1874672572451328/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827354 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787827329.411628:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c692c0 x1874672572451328/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827354 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787827329.411637:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787827329.411640:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787827329.411643:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c692c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572451328:192.168.203.129@tcp:250:kworker.0 00000100:00100000:2.0:1787827329.411648:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c6a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572451456:192.168.203.129@tcp:38:kworker.0 00000800:00000100:2.0:1787827329.411663:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787827329.411683:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827329/real 1787827329] req@ffff8cb999c6a580 x1874672572451456/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827354 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787827329.411705:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c6a580 x1874672572451456/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827354 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787827329.411710:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787827329.411713:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787827329.411715:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c6a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572451456:192.168.203.129@tcp:38:kworker.0 00000100:00100000:2.0:1787827329.411740:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c6a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572451584:192.168.203.129@tcp:38:kworker.0 00000800:00000100:2.0:1787827329.411755:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787827329.411763:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827329/real 1787827329] req@ffff8cb999c6a1c0 x1874672572451584/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827354 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787827329.411775:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c6a1c0 x1874672572451584/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827354 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787827329.411785:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787827329.411788:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787827329.411792:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c6a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572451584:192.168.203.129@tcp:38:kworker.0 00010000:00010000:2.0:1787827330.242396:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787827331.266387:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787827332.290424:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787827333.314380:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787827334.338427:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787827334.530392:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827334.530399:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787827334.530403:0:37:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827334.530412:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 286 00000100:00080000:3.0:1787827334.530421:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827334.530427:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:3.0:1787827334.530464:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb992316940 x1874672572451712/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787827334.530491:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827334.530494:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787827334.530495:0:37:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827334.530498:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 286 00000100:00080000:3.0:1787827334.530501:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827334.530505:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:3.0:1787827334.530515:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787827334.530527:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb992315e00 x1874672572451840/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787827334.530536:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827334.530539:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1787827334.530541:0:37:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827334.530547:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 286 00000100:00080000:3.0:1787827334.530550:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1787827334.530552:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb992316940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572451712:192.168.203.129@tcp:250:kworker.0 00000100:00080000:3.0:1787827334.530568:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:3.0:1787827334.530574:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787827334.530585:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb992314f00 x1874672572451968/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1787827334.530652:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787827334.530707:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827334/real 1787827334] req@ffff8cb992316940 x1874672572451712/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827359 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787827334.530728:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb992316940 x1874672572451712/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827359 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787827334.530754:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787827334.530759:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787827334.530764:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb992316940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572451712:192.168.203.129@tcp:250:kworker.0 00000100:00100000:2.0:1787827334.530794:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb992315e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572451840:192.168.203.129@tcp:38:kworker.0 00000800:00000100:2.0:1787827334.530823:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787827334.530837:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827334/real 1787827334] req@ffff8cb992315e00 x1874672572451840/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827359 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787827334.530846:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb992315e00 x1874672572451840/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827359 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787827334.530852:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787827334.530855:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787827334.530857:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb992315e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572451840:192.168.203.129@tcp:38:kworker.0 00000100:00100000:2.0:1787827334.530863:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb992314f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572451968:192.168.203.129@tcp:38:kworker.0 00000800:00000100:2.0:1787827334.530887:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787827334.530899:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827334/real 1787827334] req@ffff8cb992314f00 x1874672572451968/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827359 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787827334.530914:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb992314f00 x1874672572451968/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827359 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787827334.530921:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787827334.530924:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787827334.530926:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb992314f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572451968:192.168.203.129@tcp:38:kworker.0 00010000:00010000:1.0:1787827335.362473:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787827336.386462:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787827337.410397:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787827338.434394:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787827339.458440:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787827339.650358:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827339.650366:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787827339.650370:0:37:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827339.650378:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 291 00000100:00080000:3.0:1787827339.650387:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827339.650393:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:3.0:1787827339.650425:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb992317480 x1874672572452096/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787827339.650447:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827339.650450:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787827339.650451:0:37:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827339.650454:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 291 00000100:00080000:3.0:1787827339.650462:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827339.650464:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:3.0:1787827339.650472:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787827339.650480:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb992316580 x1874672572452224/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787827339.650487:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827339.650490:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1787827339.650506:0:37:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827339.650509:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 291 00000100:00080000:3.0:1787827339.650512:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827339.650516:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:3.0:1787827339.650520:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787827339.650530:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb992314780 x1874672572452352/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787827339.650538:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb992317480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572452096:192.168.203.129@tcp:250:kworker.0 00000800:00000100:2.0:1787827339.650618:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787827339.650674:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827339/real 1787827339] req@ffff8cb992317480 x1874672572452096/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827364 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787827339.650690:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb992317480 x1874672572452096/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827364 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787827339.650701:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787827339.650705:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787827339.650709:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb992317480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572452096:192.168.203.129@tcp:250:kworker.0 00000100:00100000:2.0:1787827339.650717:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb992316580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572452224:192.168.203.129@tcp:38:kworker.0 00000800:00000100:2.0:1787827339.650735:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787827339.650744:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827339/real 1787827339] req@ffff8cb992316580 x1874672572452224/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827364 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787827339.650752:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb992316580 x1874672572452224/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827364 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787827339.650758:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787827339.650761:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787827339.650763:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb992316580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572452224:192.168.203.129@tcp:38:kworker.0 00000100:00100000:2.0:1787827339.650789:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb992314780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572452352:192.168.203.129@tcp:38:kworker.0 00000800:00000100:2.0:1787827339.650804:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787827339.650827:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827339/real 1787827339] req@ffff8cb992314780 x1874672572452352/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827364 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787827339.650858:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb992314780 x1874672572452352/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827364 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787827339.650867:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787827339.650872:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787827339.650875:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb992314780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572452352:192.168.203.129@tcp:38:kworker.0 00010000:00010000:3.0:1787827340.482366:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787827341.506416:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787827342.530400:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787827343.554366:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787827344.578431:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787827344.770330:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827344.770338:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787827344.770341:0:37:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827344.770349:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 296 00000100:00080000:3.0:1787827344.770358:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827344.770364:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:3.0:1787827344.770395:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb992314b40 x1874672572452480/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787827344.770417:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827344.770420:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787827344.770432:0:37:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827344.770435:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 296 00000100:00080000:3.0:1787827344.770437:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827344.770439:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:3.0:1787827344.770447:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787827344.770469:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb992314780 x1874672572452608/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787827344.770477:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827344.770480:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1787827344.770481:0:37:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827344.770484:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 296 00000100:00080000:3.0:1787827344.770488:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827344.770492:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:3.0:1787827344.770495:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787827344.770505:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb992316580 x1874672572452736/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787827344.770551:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb992314b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572452480:192.168.203.129@tcp:250:kworker.0 00000800:00000100:2.0:1787827344.770686:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787827344.770785:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827344/real 1787827344] req@ffff8cb992314b40 x1874672572452480/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827369 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787827344.770814:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb992314b40 x1874672572452480/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827369 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787827344.770829:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787827344.770834:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787827344.770839:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb992314b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572452480:192.168.203.129@tcp:250:kworker.0 00000100:00100000:2.0:1787827344.770851:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb992314780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572452608:192.168.203.129@tcp:38:kworker.0 00000800:00000100:2.0:1787827344.770883:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787827344.770902:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827344/real 1787827344] req@ffff8cb992314780 x1874672572452608/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827369 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787827344.770919:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb992314780 x1874672572452608/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827369 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787827344.770954:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787827344.770960:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787827344.770965:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb992314780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572452608:192.168.203.129@tcp:38:kworker.0 00000100:00100000:2.0:1787827344.771002:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb992316580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572452736:192.168.203.129@tcp:38:kworker.0 00000800:00000100:2.0:1787827344.771032:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787827344.771044:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827344/real 1787827344] req@ffff8cb992316580 x1874672572452736/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827369 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787827344.771056:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb992316580 x1874672572452736/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827369 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787827344.771065:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787827344.771069:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787827344.771073:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb992316580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572452736:192.168.203.129@tcp:38:kworker.0 00010000:00010000:1.0:1787827345.602376:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787827346.626370:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787827347.650417:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787827348.674392:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787827349.698377:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787827349.890366:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827349.890377:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787827349.890382:0:37:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827349.890393:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 302 00000100:00080000:3.0:1787827349.890422:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827349.890434:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:3.0:1787827349.890492:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb992317480 x1874672572452864/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787827349.890525:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827349.890529:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787827349.890531:0:37:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827349.890555:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 302 00000100:00080000:3.0:1787827349.890559:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827349.890564:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:3.0:1787827349.890577:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787827349.890601:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb992317480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572452864:192.168.203.129@tcp:250:kworker.0 00000100:00100000:3.0:1787827349.890609:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb992314f00 x1874672572452992/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787827349.890621:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827349.890624:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1787827349.890625:0:37:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827349.890630:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 302 00000100:00080000:3.0:1787827349.890633:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827349.890635:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:3.0:1787827349.890639:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787827349.890651:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb992315e00 x1874672572453120/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1787827349.890688:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787827349.890738:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827349/real 1787827349] req@ffff8cb992317480 x1874672572452864/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827374 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787827349.890755:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb992317480 x1874672572452864/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827374 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787827349.890765:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787827349.890769:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787827349.890780:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb992317480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572452864:192.168.203.129@tcp:250:kworker.0 00000100:00100000:2.0:1787827349.890808:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb992314f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572452992:192.168.203.129@tcp:38:kworker.0 00000800:00000100:2.0:1787827349.890829:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787827349.890840:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827349/real 1787827349] req@ffff8cb992314f00 x1874672572452992/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827374 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787827349.890850:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb992314f00 x1874672572452992/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827374 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787827349.890856:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787827349.890859:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787827349.890862:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb992314f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572452992:192.168.203.129@tcp:38:kworker.0 00000100:00100000:2.0:1787827349.890866:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb992315e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572453120:192.168.203.129@tcp:38:kworker.0 00000800:00000100:2.0:1787827349.890879:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787827349.890885:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827349/real 1787827349] req@ffff8cb992315e00 x1874672572453120/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827374 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787827349.890894:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb992315e00 x1874672572453120/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827374 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787827349.890900:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787827349.890902:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787827349.890905:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb992315e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572453120:192.168.203.129@tcp:38:kworker.0 00010000:00010000:2.0:1787827350.722380:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787827351.746373:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787827352.770353:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787827353.794415:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787827354.818364:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787827355.010368:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827355.010379:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787827355.010384:0:37:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827355.010394:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 307 00000100:00080000:3.0:1787827355.010406:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827355.010413:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:3.0:1787827355.010446:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb992316940 x1874672572453248/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787827355.010499:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827355.010502:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787827355.010503:0:37:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827355.010506:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 307 00000100:00080000:3.0:1787827355.010509:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827355.010512:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:3.0:1787827355.010519:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787827355.010528:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb9923170c0 x1874672572453376/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787827355.010535:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827355.010538:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1787827355.010540:0:37:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827355.010542:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 307 00000100:00080000:3.0:1787827355.010544:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827355.010546:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:3.0:1787827355.010549:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787827355.010555:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb9923161c0 x1874672572453504/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787827355.010561:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb992316940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572453248:192.168.203.129@tcp:250:kworker.0 00000800:00000100:2.0:1787827355.010625:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787827355.010663:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827355/real 1787827355] req@ffff8cb992316940 x1874672572453248/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827380 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787827355.010682:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb992316940 x1874672572453248/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827380 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787827355.010694:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787827355.010699:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787827355.010703:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb992316940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572453248:192.168.203.129@tcp:250:kworker.0 00000100:00100000:2.0:1787827355.010709:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb9923170c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572453376:192.168.203.129@tcp:38:kworker.0 00000800:00000100:2.0:1787827355.010727:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787827355.010745:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827355/real 1787827355] req@ffff8cb9923170c0 x1874672572453376/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827380 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787827355.010753:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb9923170c0 x1874672572453376/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827380 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787827355.010759:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787827355.010761:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787827355.010763:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb9923170c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572453376:192.168.203.129@tcp:38:kworker.0 00000100:00100000:2.0:1787827355.010783:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb9923161c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572453504:192.168.203.129@tcp:38:kworker.0 00000800:00000100:2.0:1787827355.010795:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787827355.010821:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827355/real 1787827355] req@ffff8cb9923161c0 x1874672572453504/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827380 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787827355.010829:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb9923161c0 x1874672572453504/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827380 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787827355.010834:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787827355.010836:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787827355.010838:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb9923161c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572453504:192.168.203.129@tcp:38:kworker.0 00010000:00010000:3.0:1787827355.842350:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787827356.866446:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787827357.890385:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787827358.914409:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787827359.938449:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787827360.130431:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827360.130448:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787827360.130455:0:37:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827360.130518:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 312 00000100:00080000:3.0:1787827360.130535:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827360.130547:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:3.0:1787827360.130603:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb992315a40 x1874672572453632/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787827360.130647:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827360.130668:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787827360.130670:0:37:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827360.130675:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 312 00000100:00080000:3.0:1787827360.130680:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827360.130685:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:3.0:1787827360.130700:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787827360.130740:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb9923161c0 x1874672572453760/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787827360.130753:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1787827360.130756:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb992315a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572453632:192.168.203.129@tcp:250:kworker.0 00000100:00080000:3.0:1787827360.130759:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1787827360.130765:0:37:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827360.130770:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 312 00000100:00080000:3.0:1787827360.130774:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827360.130780:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:3.0:1787827360.130801:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787827360.130821:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb9923170c0 x1874672572453888/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1787827360.130870:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787827360.130931:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827360/real 1787827360] req@ffff8cb992315a40 x1874672572453632/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827385 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787827360.130953:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb992315a40 x1874672572453632/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827385 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787827360.130966:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787827360.130970:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787827360.130975:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb992315a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572453632:192.168.203.129@tcp:250:kworker.0 00000100:00100000:2.0:1787827360.131004:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb9923161c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572453760:192.168.203.129@tcp:38:kworker.0 00000800:00000100:2.0:1787827360.131048:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787827360.131076:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827360/real 1787827360] req@ffff8cb9923161c0 x1874672572453760/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827385 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787827360.131097:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb9923161c0 x1874672572453760/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827385 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787827360.131104:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787827360.131106:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787827360.131108:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb9923161c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572453760:192.168.203.129@tcp:38:kworker.0 00000100:00100000:2.0:1787827360.131113:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb9923170c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572453888:192.168.203.129@tcp:38:kworker.0 00000800:00000100:2.0:1787827360.131127:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787827360.131134:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827360/real 1787827360] req@ffff8cb9923170c0 x1874672572453888/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827385 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787827360.131142:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb9923170c0 x1874672572453888/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827385 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787827360.131147:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787827360.131149:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787827360.131151:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb9923170c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572453888:192.168.203.129@tcp:38:kworker.0 00010000:00010000:2.0:1787827360.962424:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787827361.986381:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787827363.010419:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787827364.034487:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787827365.058399:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787827365.250338:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827365.250347:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787827365.250351:0:37:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827365.250359:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 317 00000100:00080000:3.0:1787827365.250369:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827365.250376:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:3.0:1787827365.250420:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb992316940 x1874672572454016/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787827365.250444:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827365.250447:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787827365.250449:0:37:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827365.250451:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 317 00000100:00080000:3.0:1787827365.250454:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827365.250457:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:3.0:1787827365.250465:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787827365.250476:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb992315e00 x1874672572454144/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787827365.250487:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827365.250491:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1787827365.250493:0:37:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827365.250498:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 317 00000100:00100000:2.0:1787827365.250511:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb992316940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572454016:192.168.203.129@tcp:250:kworker.0 00000100:00080000:3.0:1787827365.250518:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827365.250527:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:3.0:1787827365.250533:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787827365.250544:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb992314f00 x1874672572454272/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1787827365.250601:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787827365.250667:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827365/real 1787827365] req@ffff8cb992316940 x1874672572454016/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827390 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787827365.250701:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb992316940 x1874672572454016/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827390 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787827365.250711:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787827365.250715:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787827365.250718:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb992316940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572454016:192.168.203.129@tcp:250:kworker.0 00000100:00100000:2.0:1787827365.250725:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb992315e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572454144:192.168.203.129@tcp:38:kworker.0 00000800:00000100:2.0:1787827365.250746:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787827365.250756:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827365/real 1787827365] req@ffff8cb992315e00 x1874672572454144/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827390 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787827365.250764:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb992315e00 x1874672572454144/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827390 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787827365.250770:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787827365.250772:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787827365.250775:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb992315e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572454144:192.168.203.129@tcp:38:kworker.0 00000100:00100000:2.0:1787827365.250801:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb992314f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572454272:192.168.203.129@tcp:38:kworker.0 00000800:00000100:2.0:1787827365.250813:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787827365.250819:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827365/real 1787827365] req@ffff8cb992314f00 x1874672572454272/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827390 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787827365.250826:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb992314f00 x1874672572454272/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827390 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787827365.250843:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787827365.250845:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787827365.250848:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb992314f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572454272:192.168.203.129@tcp:38:kworker.0 00010000:00010000:2.0:1787827366.082433:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787827367.106425:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787827368.130386:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787827369.154552:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787827370.178464:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787827370.370367:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827370.370384:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787827370.370393:0:37:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827370.370413:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 322 00000100:00080000:3.0:1787827370.370435:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827370.370450:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:3.0:1787827370.370520:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb992317480 x1874672572454400/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787827370.370567:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827370.370576:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787827370.370580:0:37:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827370.370588:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 322 00000100:00080000:3.0:1787827370.370597:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827370.370606:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:3.0:1787827370.370626:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787827370.370656:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb992316580 x1874672572454528/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787827370.370682:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1787827370.370707:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb992317480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572454400:192.168.203.129@tcp:250:kworker.0 00000100:00080000:3.0:1787827370.370712:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1787827370.370726:0:37:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827370.370742:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 322 00000100:00080000:3.0:1787827370.370755:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827370.370764:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:3.0:1787827370.370774:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787827370.370797:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb992314780 x1874672572454656/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1787827370.370816:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787827370.370925:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827370/real 1787827370] req@ffff8cb992317480 x1874672572454400/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827395 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787827370.371001:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb992317480 x1874672572454400/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827395 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787827370.371016:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787827370.371021:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787827370.371026:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb992317480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572454400:192.168.203.129@tcp:250:kworker.0 00000100:00100000:2.0:1787827370.371060:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb992316580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572454528:192.168.203.129@tcp:38:kworker.0 00000800:00000100:2.0:1787827370.371099:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787827370.371122:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827370/real 1787827370] req@ffff8cb992316580 x1874672572454528/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827395 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787827370.371145:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb992316580 x1874672572454528/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827395 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787827370.371188:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787827370.371193:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787827370.371197:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb992316580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572454528:192.168.203.129@tcp:38:kworker.0 00000100:00100000:2.0:1787827370.371205:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb992314780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572454656:192.168.203.129@tcp:38:kworker.0 00000800:00000100:2.0:1787827370.371233:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787827370.371246:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827370/real 1787827370] req@ffff8cb992314780 x1874672572454656/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827395 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787827370.371303:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb992314780 x1874672572454656/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827395 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787827370.371314:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787827370.371317:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787827370.371321:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb992314780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572454656:192.168.203.129@tcp:38:kworker.0 00010000:00010000:3.0:1787827371.202391:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787827372.226432:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787827373.250494:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787827374.274425:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787827375.298447:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787827375.490535:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827375.490544:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787827375.490549:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827375.490563:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 327 00000100:00080000:3.0:1787827375.490576:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827375.490585:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:3.0:1787827375.491365:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb992314b40 x1874672572454784/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787827375.491397:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827375.491402:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787827375.491406:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827375.491414:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 327 00000100:00080000:3.0:1787827375.491421:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827375.491428:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:3.0:1787827375.491452:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787827375.491474:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb992314780 x1874672572454912/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787827375.491489:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827375.491493:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1787827375.491496:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827375.491503:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 327 00000100:00080000:3.0:1787827375.491506:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827375.491513:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:3.0:1787827375.491522:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787827375.491539:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb992316580 x1874672572455040/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1787827375.491585:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb992314b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572454784:192.168.203.129@tcp:250:kworker.0 00000800:00000100:3.0:1787827375.491652:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:3.0:1787827375.491723:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827375/real 1787827375] req@ffff8cb992314b40 x1874672572454784/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827400 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827375.491759:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb992314b40 x1874672572454784/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827400 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827375.491780:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827375.491789:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:3.0:1787827375.491799:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb992314b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572454784:192.168.203.129@tcp:250:kworker.0 00000100:00100000:3.0:1787827375.491821:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb992314780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572454912:192.168.203.129@tcp:38:kworker.0 00000800:00000100:3.0:1787827375.491876:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:3.0:1787827375.491926:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827375/real 1787827375] req@ffff8cb992314780 x1874672572454912/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827400 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827375.491956:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb992314780 x1874672572454912/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827400 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827375.491972:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827375.491978:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:3.0:1787827375.491986:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb992314780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572454912:192.168.203.129@tcp:38:kworker.0 00000100:00100000:3.0:1787827375.492006:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb992316580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572455040:192.168.203.129@tcp:38:kworker.0 00000800:00000100:3.0:1787827375.492049:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:3.0:1787827375.492075:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827375/real 1787827375] req@ffff8cb992316580 x1874672572455040/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827400 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827375.492095:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb992316580 x1874672572455040/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827400 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827375.492107:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827375.492115:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:3.0:1787827375.492123:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb992316580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572455040:192.168.203.129@tcp:38:kworker.0 00010000:00010000:2.0:1787827376.322410:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787827377.346378:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787827378.370449:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787827379.394446:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787827380.418438:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787827380.610420:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827380.610433:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787827380.610438:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827380.610450:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 332 00000100:00080000:2.0:1787827380.610464:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827380.610473:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:2.0:1787827380.610539:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8f480 x1874672572455168/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787827380.610566:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827380.610570:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787827380.610571:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827380.610575:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 332 00000100:00080000:2.0:1787827380.610579:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827380.610590:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:2.0:1787827380.610601:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787827380.610614:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8c780 x1874672572455296/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1787827380.610619:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572455168:192.168.203.129@tcp:250:kworker.0 00000100:00080000:2.0:1787827380.610625:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827380.610631:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1787827380.610633:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827380.610636:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 332 00000100:00080000:2.0:1787827380.610639:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827380.610642:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:2.0:1787827380.610645:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787827380.610656:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8e580 x1874672572455424/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1787827380.610846:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8c780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572455296:192.168.203.129@tcp:38:kworker.0 00000100:00100000:3.0:1787827380.610910:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8e580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572455424:192.168.203.129@tcp:38:kworker.0 00010000:00010000:2.0:1787827381.442463:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787827382.466365:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787827383.490416:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:2.0:1787827383.682503:0:2381:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.129:988 00000400:00000100:2.0:1787827383.682539:0:2381:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.129@tcp at host 192.168.203.129 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:2.0:1787827383.682566:0:2381:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.129@tcp 00000800:00000100:2.0:1787827383.682596:0:2381:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.129@tcp 00000800:00000100:2.0:1787827383.682603:0:2381:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.129@tcp 00000100:00100000:3.0:1787827383.682681:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827380/real 1787827383] req@ffff8cb998a8c780 x1874672572455296/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827405 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827383.682709:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8c780 x1874672572455296/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827405 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827383.682722:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827383.682727:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:3.0:1787827383.682732:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8c780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572455296:192.168.203.129@tcp:38:kworker.0 00000100:00100000:3.0:1787827383.682740:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827380/real 1787827383] req@ffff8cb998a8e580 x1874672572455424/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827405 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827383.682752:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8e580 x1874672572455424/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827405 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827383.682761:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827383.682764:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:3.0:1787827383.682775:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8e580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572455424:192.168.203.129@tcp:38:kworker.0 00000100:00100000:3.0:1787827383.682783:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827380/real 1787827383] req@ffff8cb998a8f480 x1874672572455168/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827405 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827383.682797:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8f480 x1874672572455168/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827405 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827383.682805:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827383.682818:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:3.0:1787827383.682822:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572455168:192.168.203.129@tcp:250:kworker.0 00010000:00010000:2.0:1787827384.514820:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787827385.539331:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787827385.730445:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787827385.730469:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787827385.730474:0:37:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787827385.730484:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 337 00000100:00080000:0.0:1787827385.730495:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787827385.730503:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:0.0:1787827385.730537:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c68000 x1874672572455552/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787827385.730564:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787827385.730568:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787827385.730569:0:37:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787827385.730572:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 337 00000100:00080000:0.0:1787827385.730592:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787827385.730599:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:0.0:1787827385.730611:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787827385.730621:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c68000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572455552:192.168.203.129@tcp:250:kworker.0 00000100:00100000:0.0:1787827385.730631:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c69680 x1874672572455680/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787827385.730647:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787827385.730651:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1787827385.730654:0:37:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787827385.730659:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 337 00000100:00080000:0.0:1787827385.730664:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787827385.730670:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:0.0:1787827385.730676:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787827385.730694:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c6b480 x1874672572455808/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1787827385.730699:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:3.0:1787827385.730797:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827385/real 1787827385] req@ffff8cb999c68000 x1874672572455552/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827410 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827385.730822:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c68000 x1874672572455552/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827410 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827385.730852:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827385.730859:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:3.0:1787827385.730864:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c68000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572455552:192.168.203.129@tcp:250:kworker.0 00000100:00100000:3.0:1787827385.730895:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c69680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572455680:192.168.203.129@tcp:38:kworker.0 00000800:00000100:3.0:1787827385.730926:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:3.0:1787827385.730941:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827385/real 1787827385] req@ffff8cb999c69680 x1874672572455680/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827410 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827385.730966:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c69680 x1874672572455680/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827410 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827385.730977:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827385.730980:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:3.0:1787827385.730985:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c69680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572455680:192.168.203.129@tcp:38:kworker.0 00000100:00100000:3.0:1787827385.730993:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c6b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572455808:192.168.203.129@tcp:38:kworker.0 00000800:00000100:3.0:1787827385.731030:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:3.0:1787827385.731043:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827385/real 1787827385] req@ffff8cb999c6b480 x1874672572455808/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827410 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827385.731065:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c6b480 x1874672572455808/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827410 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827385.731075:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827385.731080:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:3.0:1787827385.731084:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c6b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572455808:192.168.203.129@tcp:38:kworker.0 00010000:00010000:3.0:1787827386.562506:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787827387.587068:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787827388.610969:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787827389.634498:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787827390.658514:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787827390.850425:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827390.850441:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787827390.850447:0:37:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827390.850461:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 343 00000100:00080000:3.0:1787827390.850475:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827390.850484:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:3.0:1787827390.850538:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb992316580 x1874672572455936/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787827390.850579:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827390.850585:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787827390.850588:0:37:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827390.850592:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 343 00000100:00080000:3.0:1787827390.850597:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827390.850618:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:3.0:1787827390.850631:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787827390.850651:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb992316580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572455936:192.168.203.129@tcp:250:kworker.0 00000100:00100000:3.0:1787827390.850661:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb992314780 x1874672572456064/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787827390.850690:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827390.850698:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1787827390.850702:0:37:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827390.850712:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 343 00000100:00080000:3.0:1787827390.850724:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827390.850732:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000800:00000100:2.0:1787827390.850743:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000008:00100000:3.0:1787827390.850750:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787827390.850783:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb992314b40 x1874672572456192/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787827390.850821:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827390/real 1787827390] req@ffff8cb992316580 x1874672572455936/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827415 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787827390.850851:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb992316580 x1874672572455936/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827415 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787827390.850867:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787827390.850873:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787827390.850878:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb992316580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572455936:192.168.203.129@tcp:250:kworker.0 00000100:00100000:2.0:1787827390.850909:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb992314780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572456064:192.168.203.129@tcp:38:kworker.0 00000800:00000100:2.0:1787827390.850939:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787827390.850956:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827390/real 1787827390] req@ffff8cb992314780 x1874672572456064/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827415 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787827390.850973:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb992314780 x1874672572456064/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827415 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787827390.850984:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787827390.850999:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787827390.851004:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb992314780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572456064:192.168.203.129@tcp:38:kworker.0 00000100:00100000:2.0:1787827390.851013:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb992314b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572456192:192.168.203.129@tcp:38:kworker.0 00000800:00000100:2.0:1787827390.851041:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787827390.851054:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827390/real 1787827390] req@ffff8cb992314b40 x1874672572456192/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827415 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787827390.851069:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb992314b40 x1874672572456192/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827415 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787827390.851080:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787827390.851084:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787827390.851088:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb992314b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572456192:192.168.203.129@tcp:38:kworker.0 00010000:00010000:2.0:1787827391.682887:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787827392.706505:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787827393.730495:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787827394.754524:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787827395.778445:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787827395.970408:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827395.970422:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787827395.970429:0:37:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827395.970463:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 348 00000100:00080000:3.0:1787827395.970479:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827395.970490:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:3.0:1787827395.970553:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb992317480 x1874672572456320/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787827395.970598:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827395.970627:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787827395.970631:0:37:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827395.970638:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 348 00000100:00080000:3.0:1787827395.970644:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827395.970651:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:3.0:1787827395.970666:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787827395.970683:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb992317480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572456320:192.168.203.129@tcp:250:kworker.0 00000100:00100000:3.0:1787827395.970688:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb992314f00 x1874672572456448/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787827395.970710:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827395.970714:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1787827395.970716:0:37:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827395.970723:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 348 00000100:00080000:3.0:1787827395.970727:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827395.970732:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:3.0:1787827395.970738:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787827395.970753:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb992315e00 x1874672572456576/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1787827395.970831:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787827395.970911:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827395/real 1787827395] req@ffff8cb992317480 x1874672572456320/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827420 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787827395.970940:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb992317480 x1874672572456320/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827420 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787827395.970957:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787827395.970963:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787827395.970982:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb992317480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572456320:192.168.203.129@tcp:250:kworker.0 00000100:00100000:2.0:1787827395.971032:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb992314f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572456448:192.168.203.129@tcp:38:kworker.0 00000800:00000100:2.0:1787827395.971082:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787827395.971108:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827395/real 1787827395] req@ffff8cb992314f00 x1874672572456448/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827420 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787827395.971127:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb992314f00 x1874672572456448/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827420 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787827395.971137:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787827395.971142:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787827395.971146:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb992314f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572456448:192.168.203.129@tcp:38:kworker.0 00000100:00100000:2.0:1787827395.971154:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb992315e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572456576:192.168.203.129@tcp:38:kworker.0 00000800:00000100:2.0:1787827395.971181:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787827395.971194:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827395/real 1787827395] req@ffff8cb992315e00 x1874672572456576/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827420 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787827395.971208:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb992315e00 x1874672572456576/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827420 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787827395.971219:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787827395.971223:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787827395.971227:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb992315e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572456576:192.168.203.129@tcp:38:kworker.0 00010000:00010000:2.0:1787827396.802485:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787827397.826541:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787827398.850374:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787827399.874462:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787827400.898586:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787827401.090446:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827401.090464:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787827401.090472:0:37:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827401.090489:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 353 00000100:00080000:3.0:1787827401.090509:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827401.090522:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:3.0:1787827401.090684:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb992316940 x1874672572456704/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787827401.090769:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827401.090774:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787827401.090777:0:37:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827401.090800:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 353 00000100:00080000:3.0:1787827401.090806:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827401.090811:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:3.0:1787827401.090826:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787827401.090844:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb9923170c0 x1874672572456832/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787827401.090877:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827401.090882:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1787827401.090886:0:37:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827401.090894:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 353 00000100:00100000:2.0:1787827401.090905:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb992316940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572456704:192.168.203.129@tcp:250:kworker.0 00000100:00080000:3.0:1787827401.090906:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827401.090925:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:3.0:1787827401.090937:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787827401.090964:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb9923161c0 x1874672572456960/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1787827401.091032:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787827401.091101:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827401/real 1787827401] req@ffff8cb992316940 x1874672572456704/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827426 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787827401.091126:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb992316940 x1874672572456704/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827426 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787827401.091141:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787827401.091147:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787827401.091170:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb992316940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572456704:192.168.203.129@tcp:250:kworker.0 00000100:00100000:2.0:1787827401.091180:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb9923170c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572456832:192.168.203.129@tcp:38:kworker.0 00000800:00000100:2.0:1787827401.091207:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787827401.091220:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827401/real 1787827401] req@ffff8cb9923170c0 x1874672572456832/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827426 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787827401.091233:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb9923170c0 x1874672572456832/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827426 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787827401.091242:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787827401.091246:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787827401.091250:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb9923170c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572456832:192.168.203.129@tcp:38:kworker.0 00000100:00100000:2.0:1787827401.091319:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb9923161c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572456960:192.168.203.129@tcp:38:kworker.0 00000800:00000100:2.0:1787827401.091364:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787827401.091396:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827401/real 1787827401] req@ffff8cb9923161c0 x1874672572456960/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827426 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787827401.091430:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb9923161c0 x1874672572456960/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827426 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787827401.091440:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787827401.091444:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787827401.091448:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb9923161c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572456960:192.168.203.129@tcp:38:kworker.0 00010000:00010000:3.0:1787827401.922443:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787827402.946641:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787827403.970442:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787827404.994358:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787827406.018432:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787827406.210361:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827406.210380:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787827406.210408:0:37:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827406.210426:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 358 00000100:00080000:3.0:1787827406.210445:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827406.210457:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:3.0:1787827406.210516:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb992315a40 x1874672572457088/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787827406.210554:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827406.210559:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787827406.210562:0:37:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827406.210566:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 358 00000100:00080000:3.0:1787827406.210573:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827406.210579:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:3.0:1787827406.210594:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787827406.210609:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb992315a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572457088:192.168.203.129@tcp:250:kworker.0 00000100:00100000:3.0:1787827406.210618:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb9923161c0 x1874672572457216/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787827406.210644:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827406.210651:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1787827406.210655:0:37:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827406.210663:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 358 00000100:00080000:3.0:1787827406.210675:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827406.210683:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000800:00000100:2.0:1787827406.210701:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000008:00100000:3.0:1787827406.210726:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787827406.210748:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827406/real 1787827406] req@ffff8cb992315a40 x1874672572457088/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827431 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827406.210755:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb9923170c0 x1874672572457344/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787827406.210813:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb992315a40 x1874672572457088/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827431 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787827406.210877:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787827406.210884:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787827406.210891:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb992315a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572457088:192.168.203.129@tcp:250:kworker.0 00000100:00100000:2.0:1787827406.210918:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb9923161c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572457216:192.168.203.129@tcp:38:kworker.0 00000800:00000100:2.0:1787827406.210939:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787827406.210948:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827406/real 1787827406] req@ffff8cb9923161c0 x1874672572457216/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827431 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787827406.210975:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb9923161c0 x1874672572457216/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827431 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787827406.210982:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787827406.210984:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787827406.210986:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb9923161c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572457216:192.168.203.129@tcp:38:kworker.0 00000100:00100000:2.0:1787827406.210991:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb9923170c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572457344:192.168.203.129@tcp:38:kworker.0 00000800:00000100:2.0:1787827406.211006:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787827406.211013:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827406/real 1787827406] req@ffff8cb9923170c0 x1874672572457344/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827431 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787827406.211021:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb9923170c0 x1874672572457344/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827431 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787827406.211029:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787827406.211032:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787827406.211036:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb9923170c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572457344:192.168.203.129@tcp:38:kworker.0 00010000:00010000:2.0:1787827407.042413:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787827408.066454:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787827409.090458:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787827410.114598:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787827411.138483:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787827411.330495:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827411.330524:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787827411.330531:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827411.330545:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 363 00000100:00080000:2.0:1787827411.330562:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827411.330619:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:2.0:1787827411.330739:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8cb40 x1874672572457472/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787827411.330813:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827411.330819:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787827411.330822:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827411.330828:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 363 00000100:00080000:2.0:1787827411.330835:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827411.330841:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:2.0:1787827411.330858:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787827411.330879:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8c000 x1874672572457600/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787827411.330895:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827411.330904:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1787827411.330908:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1787827411.330908:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8cb40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572457472:192.168.203.129@tcp:250:kworker.0 00000100:00080000:2.0:1787827411.330930:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 363 00000100:00080000:2.0:1787827411.330937:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827411.330941:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:2.0:1787827411.330950:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787827411.330974:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8e940 x1874672572457728/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1787827411.331038:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:3.0:1787827411.331126:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827411/real 1787827411] req@ffff8cb998a8cb40 x1874672572457472/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827436 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827411.331151:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8cb40 x1874672572457472/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827436 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827411.331168:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827411.331176:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:3.0:1787827411.331184:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8cb40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572457472:192.168.203.129@tcp:250:kworker.0 00000100:00100000:3.0:1787827411.331238:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572457600:192.168.203.129@tcp:38:kworker.0 00000800:00000100:3.0:1787827411.331309:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:3.0:1787827411.331327:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827411/real 1787827411] req@ffff8cb998a8c000 x1874672572457600/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827436 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827411.331341:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8c000 x1874672572457600/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827436 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827411.331350:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827411.331353:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:3.0:1787827411.331357:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572457600:192.168.203.129@tcp:38:kworker.0 00000100:00100000:3.0:1787827411.331364:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8e940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572457728:192.168.203.129@tcp:38:kworker.0 00000800:00000100:3.0:1787827411.331387:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:3.0:1787827411.331419:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827411/real 1787827411] req@ffff8cb998a8e940 x1874672572457728/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827436 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827411.331441:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8e940 x1874672572457728/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827436 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827411.331450:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827411.331453:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:3.0:1787827411.331456:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8e940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572457728:192.168.203.129@tcp:38:kworker.0 00010000:00010000:2.0:1787827412.162555:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787827413.186523:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787827414.210542:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787827415.234515:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787827416.258454:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787827416.450368:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827416.450382:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787827416.450388:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827416.450410:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 368 00000100:00080000:2.0:1787827416.450426:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827416.450437:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:2.0:1787827416.450492:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8ed00 x1874672572457856/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787827416.450537:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827416.450543:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787827416.450545:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827416.450550:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 368 00000100:00080000:2.0:1787827416.450555:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827416.450560:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:2.0:1787827416.450574:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787827416.450595:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8f840 x1874672572457984/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787827416.450611:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827416.450628:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1787827416.450631:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827416.450639:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 368 00000100:00100000:0.0:1787827416.450647:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8ed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572457856:192.168.203.129@tcp:250:kworker.0 00000100:00080000:2.0:1787827416.450649:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827416.450664:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:2.0:1787827416.450691:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787827416.450714:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8e940 x1874672572458112/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1787827416.450757:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787827416.450833:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827416/real 1787827416] req@ffff8cb998a8ed00 x1874672572457856/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827441 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827416.450859:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8ed00 x1874672572457856/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827441 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827416.450877:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827416.450901:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787827416.450907:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8ed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572457856:192.168.203.129@tcp:250:kworker.0 00000100:00100000:0.0:1787827416.450941:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8f840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572457984:192.168.203.129@tcp:38:kworker.0 00000800:00000100:0.0:1787827416.450978:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787827416.450998:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827416/real 1787827416] req@ffff8cb998a8f840 x1874672572457984/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827441 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827416.451019:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8f840 x1874672572457984/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827441 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827416.451033:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827416.451039:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787827416.451046:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8f840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572457984:192.168.203.129@tcp:38:kworker.0 00000100:00100000:0.0:1787827416.451056:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8e940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572458112:192.168.203.129@tcp:38:kworker.0 00000800:00000100:0.0:1787827416.451094:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787827416.451117:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827416/real 1787827416] req@ffff8cb998a8e940 x1874672572458112/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827441 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827416.451153:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8e940 x1874672572458112/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827441 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827416.451162:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827416.451166:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787827416.451169:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8e940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572458112:192.168.203.129@tcp:38:kworker.0 00010000:00010000:0.0:1787827417.282562:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787827418.306482:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787827419.330454:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787827420.354477:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787827421.378368:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787827421.570369:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827421.570381:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787827421.570386:0:37:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827421.570397:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 373 00000100:00080000:3.0:1787827421.570424:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827421.570433:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:3.0:1787827421.570469:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb992316940 x1874672572458240/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787827421.570493:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827421.570497:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787827421.570499:0:37:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827421.570505:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 373 00000100:00080000:3.0:1787827421.570510:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827421.570516:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:3.0:1787827421.570527:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787827421.570532:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb992316940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572458240:192.168.203.129@tcp:250:kworker.0 00000100:00100000:3.0:1787827421.570547:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb992315e00 x1874672572458368/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787827421.570575:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827421.570591:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1787827421.570594:0:37:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827421.570598:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 373 00000100:00080000:3.0:1787827421.570603:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827421.570606:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:3.0:1787827421.570612:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000800:00000100:0.0:1787827421.570624:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:3.0:1787827421.570628:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb992314f00 x1874672572458496/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787827421.570680:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827421/real 1787827421] req@ffff8cb992316940 x1874672572458240/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827446 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827421.570701:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb992316940 x1874672572458240/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827446 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827421.570710:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827421.570713:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787827421.570716:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb992316940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572458240:192.168.203.129@tcp:250:kworker.0 00000100:00100000:0.0:1787827421.570739:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb992315e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572458368:192.168.203.129@tcp:38:kworker.0 00000800:00000100:0.0:1787827421.570758:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787827421.570768:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827421/real 1787827421] req@ffff8cb992315e00 x1874672572458368/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827446 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827421.570777:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb992315e00 x1874672572458368/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827446 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827421.570784:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827421.570786:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787827421.570789:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb992315e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572458368:192.168.203.129@tcp:38:kworker.0 00000100:00100000:0.0:1787827421.570793:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb992314f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572458496:192.168.203.129@tcp:38:kworker.0 00000800:00000100:0.0:1787827421.570804:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787827421.570811:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827421/real 1787827421] req@ffff8cb992314f00 x1874672572458496/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827446 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827421.570829:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb992314f00 x1874672572458496/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827446 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827421.570835:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827421.570838:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787827421.570840:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb992314f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572458496:192.168.203.129@tcp:38:kworker.0 00010000:00010000:2.0:1787827422.402396:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787827423.426390:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787827424.450507:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787827425.474531:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787827426.498499:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787827426.690469:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787827426.690483:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787827426.690489:0:37:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787827426.690502:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 378 00000100:00080000:0.0:1787827426.690535:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787827426.690563:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:0.0:1787827426.690660:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c6bc00 x1874672572458624/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787827426.690704:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787827426.690708:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787827426.690711:0:37:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787827426.690715:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 378 00000100:00080000:0.0:1787827426.690721:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787827426.690726:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:0.0:1787827426.690740:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787827426.690764:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c6bc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572458624:192.168.203.129@tcp:250:kworker.0 00000100:00100000:0.0:1787827426.690793:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c68f00 x1874672572458752/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787827426.690807:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787827426.690810:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1787827426.690812:0:37:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787827426.690817:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 378 00000100:00080000:0.0:1787827426.690823:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787827426.690827:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:0.0:1787827426.690832:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787827426.690844:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c69e00 x1874672572458880/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1787827426.690861:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:3.0:1787827426.690926:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827426/real 1787827426] req@ffff8cb999c6bc00 x1874672572458624/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827451 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827426.690946:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c6bc00 x1874672572458624/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827451 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827426.690959:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827426.690964:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:3.0:1787827426.690968:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c6bc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572458624:192.168.203.129@tcp:250:kworker.0 00000100:00100000:3.0:1787827426.691008:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c68f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572458752:192.168.203.129@tcp:38:kworker.0 00000800:00000100:3.0:1787827426.691049:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:3.0:1787827426.691063:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827426/real 1787827426] req@ffff8cb999c68f00 x1874672572458752/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827451 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827426.691086:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c68f00 x1874672572458752/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827451 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827426.691113:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827426.691116:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:3.0:1787827426.691119:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c68f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572458752:192.168.203.129@tcp:38:kworker.0 00000100:00100000:3.0:1787827426.691126:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c69e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572458880:192.168.203.129@tcp:38:kworker.0 00000800:00000100:3.0:1787827426.691146:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:3.0:1787827426.691156:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827426/real 1787827426] req@ffff8cb999c69e00 x1874672572458880/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827451 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827426.691167:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c69e00 x1874672572458880/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827451 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827426.691175:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827426.691178:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:3.0:1787827426.691180:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c69e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572458880:192.168.203.129@tcp:38:kworker.0 00010000:00010000:2.0:1787827427.522415:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787827428.546482:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787827429.570476:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787827430.594433:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787827431.618465:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787827431.810360:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787827431.810371:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787827431.810376:0:37:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787827431.810387:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 383 00000100:00080000:0.0:1787827431.810416:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787827431.810424:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:0.0:1787827431.810487:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c69a40 x1874672572459008/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787827431.810559:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787827431.810562:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787827431.810563:0:37:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787827431.810566:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 383 00000100:00080000:0.0:1787827431.810589:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787827431.810593:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:0.0:1787827431.810616:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787827431.810627:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c6b840 x1874672572459136/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787827431.810637:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787827431.810641:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1787827431.810643:0:37:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1787827431.810643:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c69a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572459008:192.168.203.129@tcp:250:kworker.0 00000100:00080000:0.0:1787827431.810651:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 383 00000100:00080000:0.0:1787827431.810656:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787827431.810658:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:0.0:1787827431.810662:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787827431.810672:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c69e00 x1874672572459264/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1787827431.810749:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:3.0:1787827431.810816:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827431/real 1787827431] req@ffff8cb999c69a40 x1874672572459008/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827456 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827431.810848:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c69a40 x1874672572459008/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827456 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827431.810876:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827431.810881:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:3.0:1787827431.810897:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c69a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572459008:192.168.203.129@tcp:250:kworker.0 00000100:00100000:3.0:1787827431.810927:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c6b840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572459136:192.168.203.129@tcp:38:kworker.0 00000800:00000100:3.0:1787827431.810953:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:3.0:1787827431.810975:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827431/real 1787827431] req@ffff8cb999c6b840 x1874672572459136/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827456 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827431.810988:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c6b840 x1874672572459136/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827456 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827431.810997:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827431.811001:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:3.0:1787827431.811005:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c6b840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572459136:192.168.203.129@tcp:38:kworker.0 00000100:00100000:3.0:1787827431.811012:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c69e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572459264:192.168.203.129@tcp:38:kworker.0 00000800:00000100:3.0:1787827431.811042:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:3.0:1787827431.811054:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827431/real 1787827431] req@ffff8cb999c69e00 x1874672572459264/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827456 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827431.811067:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c69e00 x1874672572459264/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827456 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827431.811076:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827431.811080:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:3.0:1787827431.811084:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c69e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572459264:192.168.203.129@tcp:38:kworker.0 00010000:00010000:0.0:1787827432.642476:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787827433.666389:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787827434.690411:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787827435.714561:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787827436.738415:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787827436.930333:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787827436.930343:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787827436.930347:0:37:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787827436.930357:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 389 00000100:00080000:0.0:1787827436.930369:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787827436.930377:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:0.0:1787827436.930411:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c68f00 x1874672572459392/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787827436.930452:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787827436.930455:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787827436.930456:0:37:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787827436.930460:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 389 00000100:00080000:0.0:1787827436.930463:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787827436.930466:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:0.0:1787827436.930485:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787827436.930497:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c6bc00 x1874672572459520/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787827436.930506:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1787827436.930509:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c68f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572459392:192.168.203.129@tcp:250:kworker.0 00000100:00080000:0.0:1787827436.930510:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1787827436.930529:0:37:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787827436.930536:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 389 00000100:00080000:0.0:1787827436.930539:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787827436.930541:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:0.0:1787827436.930546:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787827436.930555:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c6b480 x1874672572459648/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1787827436.930598:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:3.0:1787827436.930633:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827436/real 1787827436] req@ffff8cb999c68f00 x1874672572459392/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827461 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827436.930647:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c68f00 x1874672572459392/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827461 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827436.930654:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827436.930657:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:3.0:1787827436.930660:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c68f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572459392:192.168.203.129@tcp:250:kworker.0 00000100:00100000:3.0:1787827436.930681:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c6bc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572459520:192.168.203.129@tcp:38:kworker.0 00000800:00000100:3.0:1787827436.930697:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:3.0:1787827436.930705:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827436/real 1787827436] req@ffff8cb999c6bc00 x1874672572459520/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827461 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827436.930713:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c6bc00 x1874672572459520/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827461 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827436.930719:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827436.930721:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:3.0:1787827436.930723:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c6bc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572459520:192.168.203.129@tcp:38:kworker.0 00000100:00100000:3.0:1787827436.930727:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c6b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572459648:192.168.203.129@tcp:38:kworker.0 00000800:00000100:3.0:1787827436.930737:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:3.0:1787827436.930743:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827436/real 1787827436] req@ffff8cb999c6b480 x1874672572459648/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827461 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827436.930759:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c6b480 x1874672572459648/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827461 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827436.930765:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827436.930767:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:3.0:1787827436.930769:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c6b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572459648:192.168.203.129@tcp:38:kworker.0 00010000:00010000:2.0:1787827437.762435:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787827438.786415:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787827439.810605:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787827440.834391:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787827441.858521:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787827442.050521:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827442.050535:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787827442.050540:0:37:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827442.050553:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 394 00000100:00080000:3.0:1787827442.050568:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827442.050576:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:3.0:1787827442.050622:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb992317480 x1874672572459776/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787827442.050663:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827442.050667:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787827442.050670:0:37:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827442.050674:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 394 00000100:00080000:3.0:1787827442.050680:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827442.050684:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:3.0:1787827442.050695:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787827442.050710:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb992314b40 x1874672572459904/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787827442.050712:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb992317480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572459776:192.168.203.129@tcp:250:kworker.0 00000100:00080000:3.0:1787827442.050765:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827442.050767:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1787827442.050769:0:37:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827442.050774:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 394 00000100:00080000:3.0:1787827442.050778:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827442.050781:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:3.0:1787827442.050785:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787827442.050797:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb992314780 x1874672572460032/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1787827442.050808:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787827442.050866:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827442/real 1787827442] req@ffff8cb992317480 x1874672572459776/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827467 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827442.050889:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb992317480 x1874672572459776/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827467 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827442.050899:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827442.050903:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787827442.050907:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb992317480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572459776:192.168.203.129@tcp:250:kworker.0 00000100:00100000:0.0:1787827442.050932:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb992314b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572459904:192.168.203.129@tcp:38:kworker.0 00000800:00000100:0.0:1787827442.050950:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787827442.050960:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827442/real 1787827442] req@ffff8cb992314b40 x1874672572459904/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827467 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827442.050970:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb992314b40 x1874672572459904/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827467 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827442.050976:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827442.050979:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787827442.050981:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb992314b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572459904:192.168.203.129@tcp:38:kworker.0 00000100:00100000:0.0:1787827442.050986:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb992314780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572460032:192.168.203.129@tcp:38:kworker.0 00000800:00000100:0.0:1787827442.050997:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787827442.051004:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827442/real 1787827442] req@ffff8cb992314780 x1874672572460032/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827467 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827442.051012:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb992314780 x1874672572460032/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827467 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827442.051029:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827442.051032:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787827442.051035:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb992314780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572460032:192.168.203.129@tcp:38:kworker.0 00010000:00010000:2.0:1787827442.882518:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787827443.906408:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787827444.930430:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787827445.954469:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787827446.978433:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787827447.170330:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827447.170338:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787827447.170341:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827447.170349:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 399 00000100:00080000:2.0:1787827447.170358:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827447.170363:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:2.0:1787827447.170392:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8c000 x1874672572460160/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787827447.170435:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827447.170438:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787827447.170439:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827447.170442:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 399 00000100:00080000:2.0:1787827447.170446:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827447.170450:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:2.0:1787827447.170460:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787827447.170479:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8cb40 x1874672572460288/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787827447.170480:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572460160:192.168.203.129@tcp:250:kworker.0 00000100:00080000:2.0:1787827447.170498:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827447.170501:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1787827447.170503:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827447.170507:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 399 00000100:00080000:2.0:1787827447.170510:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827447.170513:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:2.0:1787827447.170518:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787827447.170529:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8f480 x1874672572460416/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787827447.170580:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8cb40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572460288:192.168.203.129@tcp:38:kworker.0 00000100:00100000:0.0:1787827447.170603:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572460416:192.168.203.129@tcp:38:kworker.0 00010000:00010000:0.0:1787827448.002376:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787827449.026406:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787827450.051303:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:2.0:1787827450.242500:0:2382:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.129:988 00000400:00000100:2.0:1787827450.242532:0:2382:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.129@tcp at host 192.168.203.129 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:2.0:1787827450.242546:0:2382:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.129@tcp 00000800:00000100:2.0:1787827450.242595:0:2382:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.129@tcp 00000800:00000100:2.0:1787827450.242607:0:2382:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.129@tcp 00000100:00100000:0.0:1787827450.242625:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827447/real 1787827450] req@ffff8cb998a8cb40 x1874672572460288/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827472 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827450.242657:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8cb40 x1874672572460288/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827472 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827450.242668:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827450.242673:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787827450.242677:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8cb40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572460288:192.168.203.129@tcp:38:kworker.0 00000100:00100000:0.0:1787827450.242685:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827447/real 1787827450] req@ffff8cb998a8f480 x1874672572460416/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827472 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827450.242697:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8f480 x1874672572460416/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827472 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827450.242706:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827450.242719:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787827450.242723:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572460416:192.168.203.129@tcp:38:kworker.0 00000100:00100000:0.0:1787827450.242730:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827447/real 1787827450] req@ffff8cb998a8c000 x1874672572460160/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827472 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827450.242745:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8c000 x1874672572460160/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827472 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827450.242752:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827450.242755:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787827450.242758:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572460160:192.168.203.129@tcp:250:kworker.0 00010000:00010000:1.0:1787827451.074468:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787827452.098436:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787827452.290389:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827452.290398:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787827452.290403:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827452.290415:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 404 00000100:00080000:3.0:1787827452.290427:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827452.290433:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:3.0:1787827452.290463:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb992316580 x1874672572460544/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787827452.290492:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827452.290497:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787827452.290499:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827452.290505:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 404 00000100:00080000:3.0:1787827452.290510:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827452.290515:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:0.0:1787827452.290525:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb992316580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572460544:192.168.203.129@tcp:250:kworker.0 00000008:00100000:3.0:1787827452.290538:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787827452.290557:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb992314780 x1874672572460672/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787827452.290570:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827452.290574:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1787827452.290576:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827452.290581:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 404 00000100:00080000:3.0:1787827452.290587:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827452.290591:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:3.0:1787827452.290597:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000800:00000100:0.0:1787827452.290608:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:3.0:1787827452.290613:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb992314b40 x1874672572460800/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787827452.290663:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827452/real 1787827452] req@ffff8cb992316580 x1874672572460544/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827477 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827452.290685:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb992316580 x1874672572460544/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827477 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827452.290695:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827452.290700:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787827452.290705:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb992316580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572460544:192.168.203.129@tcp:250:kworker.0 00000100:00100000:0.0:1787827452.290738:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb992314780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572460672:192.168.203.129@tcp:38:kworker.0 00000800:00000100:0.0:1787827452.290766:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787827452.290780:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827452/real 1787827452] req@ffff8cb992314780 x1874672572460672/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827477 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827452.290809:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb992314780 x1874672572460672/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827477 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827452.290819:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827452.290822:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787827452.290827:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb992314780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572460672:192.168.203.129@tcp:38:kworker.0 00000100:00100000:0.0:1787827452.290833:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb992314b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572460800:192.168.203.129@tcp:38:kworker.0 00000800:00000100:0.0:1787827452.290855:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787827452.290867:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827452/real 1787827452] req@ffff8cb992314b40 x1874672572460800/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827477 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827452.290878:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb992314b40 x1874672572460800/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827477 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827452.290886:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827452.290889:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787827452.290892:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb992314b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572460800:192.168.203.129@tcp:38:kworker.0 00010000:00010000:2.0:1787827453.122555:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787827454.147525:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787827455.170875:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787827456.194859:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787827457.218548:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787827457.410413:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827457.410425:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787827457.410430:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827457.410440:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 409 00000100:00080000:2.0:1787827457.410452:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827457.410459:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:2.0:1787827457.410501:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8e580 x1874672572460928/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787827457.410527:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827457.410531:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787827457.410532:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827457.410535:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 409 00000100:00080000:2.0:1787827457.410538:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827457.410542:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:2.0:1787827457.410552:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787827457.410561:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8c780 x1874672572461056/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787827457.410570:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827457.410572:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00100000:0.0:1787827457.410587:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8e580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572460928:192.168.203.129@tcp:250:kworker.0 00000100:00080000:2.0:1787827457.410596:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827457.410603:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 409 00000100:00080000:2.0:1787827457.410606:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827457.410609:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:2.0:1787827457.410614:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787827457.410628:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8f0c0 x1874672572461184/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1787827457.410662:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787827457.410700:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827457/real 1787827457] req@ffff8cb998a8e580 x1874672572460928/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827482 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827457.410714:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8e580 x1874672572460928/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827482 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827457.410739:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827457.410742:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787827457.410745:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8e580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572460928:192.168.203.129@tcp:250:kworker.0 00000100:00100000:0.0:1787827457.410767:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8c780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572461056:192.168.203.129@tcp:38:kworker.0 00000800:00000100:0.0:1787827457.410785:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787827457.410793:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827457/real 1787827457] req@ffff8cb998a8c780 x1874672572461056/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827482 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827457.410801:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8c780 x1874672572461056/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827482 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827457.410806:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827457.410808:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787827457.410810:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8c780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572461056:192.168.203.129@tcp:38:kworker.0 00000100:00100000:0.0:1787827457.410814:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8f0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572461184:192.168.203.129@tcp:38:kworker.0 00000800:00000100:0.0:1787827457.410825:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787827457.410831:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827457/real 1787827457] req@ffff8cb998a8f0c0 x1874672572461184/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827482 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827457.410838:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8f0c0 x1874672572461184/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827482 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827457.410843:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827457.410845:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787827457.410847:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8f0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572461184:192.168.203.129@tcp:38:kworker.0 00010000:00010000:2.0:1787827458.242875:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787827459.266655:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787827460.290422:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787827461.314415:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787827462.338445:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787827462.530392:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827462.530400:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787827462.530404:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827462.530413:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 414 00000100:00080000:2.0:1787827462.530424:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827462.530429:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:2.0:1787827462.530463:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8cf00 x1874672572461312/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787827462.530484:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827462.530487:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787827462.530488:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827462.530490:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 414 00000100:00080000:2.0:1787827462.530492:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827462.530495:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:2.0:1787827462.530517:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787827462.530526:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8d680 x1874672572461440/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787827462.530534:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827462.530538:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00100000:0.0:1787827462.530539:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8cf00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572461312:192.168.203.129@tcp:250:kworker.0 00000100:00080000:2.0:1787827462.530540:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827462.530547:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 414 00000100:00080000:2.0:1787827462.530552:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827462.530555:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:2.0:1787827462.530558:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787827462.530567:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8e1c0 x1874672572461568/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1787827462.530603:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787827462.530636:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827462/real 1787827462] req@ffff8cb998a8cf00 x1874672572461312/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827487 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827462.530650:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8cf00 x1874672572461312/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827487 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827462.530658:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827462.530660:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787827462.530663:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8cf00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572461312:192.168.203.129@tcp:250:kworker.0 00000100:00100000:0.0:1787827462.530682:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8d680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572461440:192.168.203.129@tcp:38:kworker.0 00000800:00000100:0.0:1787827462.530697:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787827462.530705:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827462/real 1787827462] req@ffff8cb998a8d680 x1874672572461440/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827487 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827462.530713:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8d680 x1874672572461440/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827487 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827462.530718:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827462.530720:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787827462.530722:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8d680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572461440:192.168.203.129@tcp:38:kworker.0 00000100:00100000:0.0:1787827462.530726:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8e1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572461568:192.168.203.129@tcp:38:kworker.0 00000800:00000100:0.0:1787827462.530736:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787827462.530742:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827462/real 1787827462] req@ffff8cb998a8e1c0 x1874672572461568/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827487 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827462.530756:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8e1c0 x1874672572461568/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827487 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827462.530762:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827462.530764:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787827462.530766:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8e1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572461568:192.168.203.129@tcp:38:kworker.0 00010000:00010000:3.0:1787827463.362395:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787827464.386461:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787827465.410491:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787827466.434488:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787827467.458408:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787827467.650361:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787827467.650371:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787827467.650375:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787827467.650385:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 419 00000100:00080000:0.0:1787827467.650397:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787827467.650404:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:0.0:1787827467.650441:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c69680 x1874672572461696/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787827467.650473:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787827467.650482:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787827467.650484:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787827467.650489:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 419 00000100:00080000:0.0:1787827467.650493:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787827467.650498:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:0.0:1787827467.650508:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787827467.650515:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c69680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572461696:192.168.203.129@tcp:250:kworker.0 00000100:00100000:0.0:1787827467.650526:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c68000 x1874672572461824/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787827467.650559:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787827467.650563:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1787827467.650565:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787827467.650570:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 419 00000100:00080000:0.0:1787827467.650575:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000800:00000100:3.0:1787827467.650577:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00080000:0.0:1787827467.650583:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:0.0:1787827467.650590:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787827467.650605:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c6a1c0 x1874672572461952/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1787827467.650615:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827467/real 1787827467] req@ffff8cb999c69680 x1874672572461696/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827492 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827467.650630:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c69680 x1874672572461696/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827492 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827467.650638:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827467.650647:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:3.0:1787827467.650650:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c69680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572461696:192.168.203.129@tcp:250:kworker.0 00000100:00100000:3.0:1787827467.650679:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c68000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572461824:192.168.203.129@tcp:38:kworker.0 00000800:00000100:3.0:1787827467.650704:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:3.0:1787827467.650719:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827467/real 1787827467] req@ffff8cb999c68000 x1874672572461824/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827492 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827467.650730:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c68000 x1874672572461824/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827492 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827467.650736:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827467.650739:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:3.0:1787827467.650741:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c68000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572461824:192.168.203.129@tcp:38:kworker.0 00000100:00100000:3.0:1787827467.650745:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c6a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572461952:192.168.203.129@tcp:38:kworker.0 00000800:00000100:3.0:1787827467.650762:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:3.0:1787827467.650784:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827467/real 1787827467] req@ffff8cb999c6a1c0 x1874672572461952/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827492 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827467.650797:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c6a1c0 x1874672572461952/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827492 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827467.650805:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827467.650808:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:3.0:1787827467.650812:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c6a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572461952:192.168.203.129@tcp:38:kworker.0 00010000:00010000:2.0:1787827468.482412:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787827469.506388:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787827470.530454:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787827471.554456:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787827472.578351:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787827472.770363:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787827472.770376:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787827472.770381:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787827472.770394:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 424 00000100:00080000:0.0:1787827472.770408:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787827472.770417:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:0.0:1787827472.770466:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c6a580 x1874672572462080/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787827472.770498:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787827472.770502:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787827472.770504:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787827472.770509:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 424 00000100:00080000:0.0:1787827472.770514:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787827472.770534:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:3.0:1787827472.770544:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c6a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572462080:192.168.203.129@tcp:250:kworker.0 00000008:00100000:0.0:1787827472.770551:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787827472.770574:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c692c0 x1874672572462208/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787827472.770588:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787827472.770592:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1787827472.770595:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787827472.770600:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 424 00000100:00080000:0.0:1787827472.770604:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787827472.770609:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:0.0:1787827472.770616:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000800:00000100:3.0:1787827472.770634:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787827472.770635:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c6a1c0 x1874672572462336/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1787827472.770686:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827472/real 1787827472] req@ffff8cb999c6a580 x1874672572462080/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827497 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827472.770703:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c6a580 x1874672572462080/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827497 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827472.770716:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827472.770721:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:3.0:1787827472.770725:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c6a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572462080:192.168.203.129@tcp:250:kworker.0 00000100:00100000:3.0:1787827472.770758:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c692c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572462208:192.168.203.129@tcp:38:kworker.0 00000800:00000100:3.0:1787827472.770778:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:3.0:1787827472.770787:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827472/real 1787827472] req@ffff8cb999c692c0 x1874672572462208/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827497 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827472.770796:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c692c0 x1874672572462208/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827497 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827472.770801:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827472.770804:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:3.0:1787827472.770806:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c692c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572462208:192.168.203.129@tcp:38:kworker.0 00000100:00100000:3.0:1787827472.770810:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c6a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572462336:192.168.203.129@tcp:38:kworker.0 00000800:00000100:3.0:1787827472.770821:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:3.0:1787827472.770827:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827472/real 1787827472] req@ffff8cb999c6a1c0 x1874672572462336/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827497 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827472.770866:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c6a1c0 x1874672572462336/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827497 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827472.770875:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827472.770878:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:3.0:1787827472.770882:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c6a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572462336:192.168.203.129@tcp:38:kworker.0 00010000:00010000:2.0:1787827473.602511:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787827474.626423:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787827475.650365:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787827476.674375:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787827477.698410:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787827477.890377:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787827477.890387:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787827477.890391:0:37:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787827477.890401:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 430 00000100:00080000:1.0:1787827477.890413:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787827477.890420:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:1.0:1787827477.890458:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f392c0 x1874672572462464/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787827477.890502:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787827477.890506:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787827477.890508:0:37:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787827477.890513:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 430 00000100:00080000:1.0:1787827477.890517:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787827477.890523:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:1.0:1787827477.890532:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787827477.890548:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f3b480 x1874672572462592/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1787827477.890554:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f392c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572462464:192.168.203.129@tcp:250:kworker.0 00000100:00080000:1.0:1787827477.890564:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787827477.890569:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1787827477.890571:0:37:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787827477.890576:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 430 00000100:00080000:1.0:1787827477.890580:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787827477.890585:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:1.0:1787827477.890590:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787827477.890604:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f3a580 x1874672572462720/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1787827477.890621:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:3.0:1787827477.890664:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827477/real 1787827477] req@ffff8cb999f392c0 x1874672572462464/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827502 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827477.890678:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f392c0 x1874672572462464/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827502 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827477.890685:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827477.890688:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:3.0:1787827477.890691:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f392c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572462464:192.168.203.129@tcp:250:kworker.0 00000100:00100000:3.0:1787827477.890710:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f3b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572462592:192.168.203.129@tcp:38:kworker.0 00000800:00000100:3.0:1787827477.890726:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:3.0:1787827477.890735:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827477/real 1787827477] req@ffff8cb999f3b480 x1874672572462592/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827502 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827477.890742:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f3b480 x1874672572462592/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827502 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827477.890759:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827477.890761:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:3.0:1787827477.890764:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f3b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572462592:192.168.203.129@tcp:38:kworker.0 00000100:00100000:3.0:1787827477.890768:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f3a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572462720:192.168.203.129@tcp:38:kworker.0 00000800:00000100:3.0:1787827477.890783:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:3.0:1787827477.890790:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827477/real 1787827477] req@ffff8cb999f3a580 x1874672572462720/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827502 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827477.890797:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f3a580 x1874672572462720/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827502 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827477.890803:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827477.890804:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:3.0:1787827477.890806:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f3a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572462720:192.168.203.129@tcp:38:kworker.0 00010000:00010000:3.0:1787827478.722371:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787827479.746384:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787827480.770456:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787827481.794356:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787827482.818407:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787827483.010353:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827483.010362:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787827483.010366:0:37:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827483.010375:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 435 00000100:00080000:2.0:1787827483.010384:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827483.010407:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:2.0:1787827483.010438:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8da40 x1874672572462848/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787827483.010469:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827483.010474:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787827483.010477:0:37:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827483.010481:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 435 00000100:00080000:2.0:1787827483.010487:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827483.010503:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:2.0:1787827483.010514:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787827483.010527:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8fc00 x1874672572462976/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1787827483.010530:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8da40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572462848:192.168.203.129@tcp:250:kworker.0 00000100:00080000:2.0:1787827483.010540:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827483.010543:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1787827483.010545:0:37:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827483.010549:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 435 00000100:00080000:2.0:1787827483.010552:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827483.010565:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:2.0:1787827483.010569:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787827483.010580:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8c3c0 x1874672572463104/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1787827483.010592:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:3.0:1787827483.010645:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827483/real 1787827483] req@ffff8cb998a8da40 x1874672572462848/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827508 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827483.010660:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8da40 x1874672572462848/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827508 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827483.010669:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827483.010682:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:3.0:1787827483.010686:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8da40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572462848:192.168.203.129@tcp:250:kworker.0 00000100:00100000:3.0:1787827483.010713:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8fc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572462976:192.168.203.129@tcp:38:kworker.0 00000800:00000100:3.0:1787827483.010742:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:3.0:1787827483.010759:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827483/real 1787827483] req@ffff8cb998a8fc00 x1874672572462976/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827508 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827483.010773:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8fc00 x1874672572462976/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827508 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827483.010780:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827483.010782:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:3.0:1787827483.010785:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8fc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572462976:192.168.203.129@tcp:38:kworker.0 00000100:00100000:3.0:1787827483.010790:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8c3c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572463104:192.168.203.129@tcp:38:kworker.0 00000800:00000100:3.0:1787827483.010808:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:3.0:1787827483.010821:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827483/real 1787827483] req@ffff8cb998a8c3c0 x1874672572463104/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827508 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827483.010835:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8c3c0 x1874672572463104/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827508 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827483.010845:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827483.010849:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:3.0:1787827483.010853:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8c3c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572463104:192.168.203.129@tcp:38:kworker.0 00010000:00010000:0.0:1787827483.842407:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787827484.866396:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787827485.890385:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787827486.914422:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787827487.938398:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787827488.130397:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827488.130419:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787827488.130423:0:37:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827488.130435:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 440 00000100:00080000:2.0:1787827488.130446:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827488.130453:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:2.0:1787827488.130489:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8d2c0 x1874672572463232/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787827488.130519:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827488.130523:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787827488.130526:0:37:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827488.130531:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 440 00000100:00080000:2.0:1787827488.130538:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827488.130544:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:2.0:1787827488.130559:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787827488.130585:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8de00 x1874672572463360/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1787827488.130605:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8d2c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572463232:192.168.203.129@tcp:250:kworker.0 00000100:00080000:2.0:1787827488.130610:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827488.130615:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1787827488.130619:0:37:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827488.130629:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 440 00000100:00080000:2.0:1787827488.130635:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827488.130640:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:2.0:1787827488.130650:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787827488.130671:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8c3c0 x1874672572463488/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1787827488.130693:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:3.0:1787827488.130851:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827488/real 1787827488] req@ffff8cb998a8d2c0 x1874672572463232/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827513 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827488.130996:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8d2c0 x1874672572463232/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827513 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827488.131007:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827488.131041:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:3.0:1787827488.131046:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8d2c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572463232:192.168.203.129@tcp:250:kworker.0 00000100:00100000:3.0:1787827488.131085:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8de00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572463360:192.168.203.129@tcp:38:kworker.0 00000800:00000100:3.0:1787827488.131135:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:3.0:1787827488.131171:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827488/real 1787827488] req@ffff8cb998a8de00 x1874672572463360/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827513 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827488.131182:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8de00 x1874672572463360/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827513 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827488.131188:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827488.131190:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:3.0:1787827488.131193:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8de00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572463360:192.168.203.129@tcp:38:kworker.0 00000100:00100000:3.0:1787827488.131198:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8c3c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572463488:192.168.203.129@tcp:38:kworker.0 00000800:00000100:3.0:1787827488.131212:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:3.0:1787827488.131219:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827488/real 1787827488] req@ffff8cb998a8c3c0 x1874672572463488/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827513 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827488.131241:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8c3c0 x1874672572463488/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827513 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827488.131246:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827488.131249:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:3.0:1787827488.131251:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8c3c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572463488:192.168.203.129@tcp:38:kworker.0 00010000:00010000:2.0:1787827488.962383:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787827489.986455:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787827491.010334:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787827492.034488:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787827493.058554:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787827493.250462:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827493.250479:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787827493.250485:0:37:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827493.250502:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 445 00000100:00080000:2.0:1787827493.250520:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827493.250549:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:2.0:1787827493.250605:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8fc00 x1874672572463616/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787827493.250639:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827493.250643:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787827493.250645:0:37:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827493.250648:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 445 00000100:00080000:2.0:1787827493.250652:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827493.250657:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:2.0:1787827493.250672:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787827493.250692:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8da40 x1874672572463744/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787827493.250711:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827493.250717:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1787827493.250720:0:37:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827493.250733:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 445 00000100:00100000:3.0:1787827493.250750:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8fc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572463616:192.168.203.129@tcp:250:kworker.0 00000100:00080000:2.0:1787827493.250770:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827493.250781:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:2.0:1787827493.250788:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787827493.250806:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8e1c0 x1874672572463872/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1787827493.250857:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:3.0:1787827493.250916:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827493/real 1787827493] req@ffff8cb998a8fc00 x1874672572463616/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827518 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827493.250961:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8fc00 x1874672572463616/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827518 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827493.250984:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827493.250993:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:3.0:1787827493.251002:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8fc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572463616:192.168.203.129@tcp:250:kworker.0 00000100:00100000:3.0:1787827493.251043:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8da40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572463744:192.168.203.129@tcp:38:kworker.0 00000800:00000100:3.0:1787827493.251079:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:3.0:1787827493.251126:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827493/real 1787827493] req@ffff8cb998a8da40 x1874672572463744/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827518 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827493.251144:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8da40 x1874672572463744/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827518 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827493.251190:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827493.251196:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:3.0:1787827493.251201:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8da40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572463744:192.168.203.129@tcp:38:kworker.0 00000100:00100000:3.0:1787827493.251210:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8e1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572463872:192.168.203.129@tcp:38:kworker.0 00000800:00000100:3.0:1787827493.251242:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:3.0:1787827493.251325:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827493/real 1787827493] req@ffff8cb998a8e1c0 x1874672572463872/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827518 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827493.251357:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8e1c0 x1874672572463872/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827518 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827493.251389:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827493.251397:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:3.0:1787827493.251404:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8e1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572463872:192.168.203.129@tcp:38:kworker.0 00010000:00010000:3.0:1787827494.082459:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787827495.106409:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787827496.130473:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787827497.154365:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787827498.178430:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787827498.370422:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827498.370432:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787827498.370436:0:37:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827498.370447:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 450 00000100:00080000:2.0:1787827498.370461:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827498.370470:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:2.0:1787827498.370513:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8d680 x1874672572464000/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787827498.370543:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827498.370547:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787827498.370548:0:37:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827498.370552:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 450 00000100:00080000:2.0:1787827498.370555:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827498.370559:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:2.0:1787827498.370567:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787827498.370578:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8cf00 x1874672572464128/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787827498.370608:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827498.370611:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1787827498.370613:0:37:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827498.370617:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 450 00000100:00080000:2.0:1787827498.370620:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827498.370625:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:2.0:1787827498.370629:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787827498.370641:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8f0c0 x1874672572464256/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1787827498.370650:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8d680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572464000:192.168.203.129@tcp:250:kworker.0 00000800:00000100:3.0:1787827498.370776:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:3.0:1787827498.370859:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827498/real 1787827498] req@ffff8cb998a8d680 x1874672572464000/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827523 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827498.370883:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8d680 x1874672572464000/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827523 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827498.370897:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827498.370918:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:3.0:1787827498.370926:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8d680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572464000:192.168.203.129@tcp:250:kworker.0 00000100:00100000:3.0:1787827498.370938:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8cf00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572464128:192.168.203.129@tcp:38:kworker.0 00000800:00000100:3.0:1787827498.370968:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:3.0:1787827498.370984:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827498/real 1787827498] req@ffff8cb998a8cf00 x1874672572464128/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827523 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827498.370998:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8cf00 x1874672572464128/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827523 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827498.371007:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827498.371011:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:3.0:1787827498.371014:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8cf00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572464128:192.168.203.129@tcp:38:kworker.0 00000100:00100000:3.0:1787827498.371067:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8f0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572464256:192.168.203.129@tcp:38:kworker.0 00000800:00000100:3.0:1787827498.371105:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:3.0:1787827498.371119:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827498/real 1787827498] req@ffff8cb998a8f0c0 x1874672572464256/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827523 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827498.371133:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8f0c0 x1874672572464256/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827523 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827498.371154:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827498.371157:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:3.0:1787827498.371161:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8f0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572464256:192.168.203.129@tcp:38:kworker.0 00010000:00010000:0.0:1787827499.202456:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787827500.226360:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787827501.250497:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787827502.274434:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787827503.298424:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787827503.490403:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827503.490418:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787827503.490423:0:37:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827503.490437:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 455 00000100:00080000:2.0:1787827503.490450:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827503.490459:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:2.0:1787827503.490502:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8c780 x1874672572464384/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787827503.490526:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827503.490529:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787827503.490531:0:37:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827503.490533:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 455 00000100:00080000:2.0:1787827503.490536:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827503.490538:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:2.0:1787827503.490563:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787827503.490573:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8e580 x1874672572464512/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787827503.490583:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1787827503.490583:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8c780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572464384:192.168.203.129@tcp:250:kworker.0 00000100:00080000:2.0:1787827503.490589:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1787827503.490592:0:37:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827503.490596:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 455 00000100:00080000:2.0:1787827503.490598:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827503.490601:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:2.0:1787827503.490604:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787827503.490615:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8c000 x1874672572464640/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1787827503.490662:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:3.0:1787827503.490725:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827503/real 1787827503] req@ffff8cb998a8c780 x1874672572464384/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827528 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827503.490744:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8c780 x1874672572464384/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827528 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827503.490754:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827503.490757:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:3.0:1787827503.490760:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8c780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572464384:192.168.203.129@tcp:250:kworker.0 00000100:00100000:3.0:1787827503.490785:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8e580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572464512:192.168.203.129@tcp:38:kworker.0 00000800:00000100:3.0:1787827503.490813:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:3.0:1787827503.490824:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827503/real 1787827503] req@ffff8cb998a8e580 x1874672572464512/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827528 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827503.490832:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8e580 x1874672572464512/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827528 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827503.490838:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827503.490841:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:3.0:1787827503.490843:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8e580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572464512:192.168.203.129@tcp:38:kworker.0 00000100:00100000:3.0:1787827503.490847:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572464640:192.168.203.129@tcp:38:kworker.0 00000800:00000100:3.0:1787827503.490859:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:3.0:1787827503.490865:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827503/real 1787827503] req@ffff8cb998a8c000 x1874672572464640/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827528 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827503.490882:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8c000 x1874672572464640/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827528 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827503.490888:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827503.490890:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:3.0:1787827503.490892:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572464640:192.168.203.129@tcp:38:kworker.0 00010000:00010000:0.0:1787827504.322455:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787827505.346471:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787827506.370408:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787827507.394436:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787827508.418407:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787827508.610398:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787827508.610406:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787827508.610409:0:37:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787827508.610417:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 460 00000100:00080000:1.0:1787827508.610442:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787827508.610448:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:1.0:1787827508.610478:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f3a1c0 x1874672572464768/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787827508.610502:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787827508.610504:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787827508.610506:0:37:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787827508.610508:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 460 00000100:00080000:1.0:1787827508.610510:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787827508.610513:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:1.0:1787827508.610520:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787827508.610528:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f383c0 x1874672572464896/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787827508.610535:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787827508.610539:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1787827508.610540:0:37:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787827508.610542:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 460 00000100:00080000:1.0:1787827508.610544:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787827508.610548:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:1.0:1787827508.610553:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787827508.610559:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f3a580 x1874672572465024/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1787827508.610584:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f3a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572464768:192.168.203.129@tcp:250:kworker.0 00000800:00000100:3.0:1787827508.610680:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:3.0:1787827508.610753:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827508/real 1787827508] req@ffff8cb999f3a1c0 x1874672572464768/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827533 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827508.610778:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f3a1c0 x1874672572464768/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827533 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827508.610788:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827508.610792:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:3.0:1787827508.610795:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f3a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572464768:192.168.203.129@tcp:250:kworker.0 00000100:00100000:3.0:1787827508.610802:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f383c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572464896:192.168.203.129@tcp:38:kworker.0 00000800:00000100:3.0:1787827508.610822:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:3.0:1787827508.610834:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827508/real 1787827508] req@ffff8cb999f383c0 x1874672572464896/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827533 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827508.610847:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f383c0 x1874672572464896/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827533 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827508.610877:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827508.610881:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:3.0:1787827508.610884:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f383c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572464896:192.168.203.129@tcp:38:kworker.0 00000100:00100000:3.0:1787827508.610917:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f3a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572465024:192.168.203.129@tcp:38:kworker.0 00000800:00000100:3.0:1787827508.610943:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:3.0:1787827508.610954:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827508/real 1787827508] req@ffff8cb999f3a580 x1874672572465024/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827533 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827508.610963:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f3a580 x1874672572465024/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827533 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827508.610968:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827508.610971:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:3.0:1787827508.610986:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f3a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572465024:192.168.203.129@tcp:38:kworker.0 00010000:00010000:3.0:1787827509.442425:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787827510.466376:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787827511.490395:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787827512.514370:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787827513.538406:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787827513.730342:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827513.730351:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787827513.730354:0:37:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827513.730370:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 465 00000100:00080000:2.0:1787827513.730382:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827513.730388:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:2.0:1787827513.730418:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8f480 x1874672572465152/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787827513.730442:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827513.730445:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787827513.730447:0:37:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827513.730450:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 465 00000100:00080000:2.0:1787827513.730467:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827513.730470:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:2.0:1787827513.730478:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787827513.730487:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8cb40 x1874672572465280/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787827513.730495:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827513.730498:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1787827513.730499:0:37:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827513.730502:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 465 00000100:00080000:2.0:1787827513.730504:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827513.730507:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:2.0:1787827513.730510:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787827513.730529:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8e940 x1874672572465408/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1787827513.730536:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572465152:192.168.203.129@tcp:250:kworker.0 00000100:00100000:3.0:1787827513.730652:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8cb40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572465280:192.168.203.129@tcp:38:kworker.0 00000100:00100000:3.0:1787827513.730696:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8e940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572465408:192.168.203.129@tcp:38:kworker.0 00010000:00010000:0.0:1787827514.562456:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787827515.586378:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787827516.610788:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:3.0:1787827516.802749:0:2383:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.129:988 00000400:00000100:3.0:1787827516.802811:0:2383:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.129@tcp at host 192.168.203.129 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:3.0:1787827516.802836:0:2383:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.129@tcp 00000800:00000100:3.0:1787827516.802896:0:2383:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.129@tcp 00000800:00000100:3.0:1787827516.802910:0:2383:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.129@tcp 00000100:00100000:0.0:1787827516.802935:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827513/real 1787827516] req@ffff8cb998a8f480 x1874672572465152/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827538 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827516.802974:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8f480 x1874672572465152/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827538 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827516.802991:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827516.802999:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787827516.803005:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572465152:192.168.203.129@tcp:250:kworker.0 00000100:00100000:0.0:1787827516.803018:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827513/real 1787827516] req@ffff8cb998a8cb40 x1874672572465280/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827538 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827516.803035:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8cb40 x1874672572465280/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827538 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827516.803045:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827516.803050:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787827516.803056:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8cb40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572465280:192.168.203.129@tcp:38:kworker.0 00000100:00100000:0.0:1787827516.803087:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827513/real 1787827516] req@ffff8cb998a8e940 x1874672572465408/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827538 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827516.803107:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8e940 x1874672572465408/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827538 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827516.803131:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827516.803136:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787827516.803140:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8e940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572465408:192.168.203.129@tcp:38:kworker.0 00010000:00010000:2.0:1787827517.634492:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787827518.658844:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787827518.850376:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827518.850390:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787827518.850395:0:37:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827518.850414:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 471 00000100:00080000:2.0:1787827518.850427:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827518.850435:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:2.0:1787827518.850480:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8f840 x1874672572465536/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787827518.850524:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827518.850527:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787827518.850529:0:37:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827518.850534:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 471 00000100:00080000:2.0:1787827518.850540:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827518.850547:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:2.0:1787827518.850559:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787827518.850574:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8f840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572465536:192.168.203.129@tcp:250:kworker.0 00000100:00100000:2.0:1787827518.850578:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8ed00 x1874672572465664/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787827518.850600:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827518.850604:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1787827518.850605:0:37:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827518.850609:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 471 00000100:00080000:2.0:1787827518.850612:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827518.850616:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:2.0:1787827518.850621:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787827518.850632:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8e940 x1874672572465792/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1787827518.850676:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787827518.850730:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827518/real 1787827518] req@ffff8cb998a8f840 x1874672572465536/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827543 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827518.850749:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8f840 x1874672572465536/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827543 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827518.850762:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827518.850767:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787827518.850771:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8f840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572465536:192.168.203.129@tcp:250:kworker.0 00000100:00100000:0.0:1787827518.850802:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8ed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572465664:192.168.203.129@tcp:38:kworker.0 00000800:00000100:0.0:1787827518.850827:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787827518.850841:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827518/real 1787827518] req@ffff8cb998a8ed00 x1874672572465664/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827543 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827518.850854:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8ed00 x1874672572465664/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827543 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827518.850863:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827518.850866:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787827518.850870:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8ed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572465664:192.168.203.129@tcp:38:kworker.0 00000100:00100000:0.0:1787827518.850896:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8e940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572465792:192.168.203.129@tcp:38:kworker.0 00000800:00000100:0.0:1787827518.850940:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787827518.850954:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827518/real 1787827518] req@ffff8cb998a8e940 x1874672572465792/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827543 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827518.850967:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8e940 x1874672572465792/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827543 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827518.850975:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827518.850979:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787827518.850983:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8e940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572465792:192.168.203.129@tcp:38:kworker.0 00010000:00010000:2.0:1787827519.682505:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787827520.706670:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787827521.731316:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787827522.754524:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787827523.778593:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787827523.970390:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827523.970416:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787827523.970420:0:37:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827523.970429:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 476 00000100:00080000:2.0:1787827523.970439:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827523.970446:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:2.0:1787827523.970479:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8cb40 x1874672572465920/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787827523.970509:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827523.970513:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787827523.970515:0:37:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827523.970519:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 476 00000100:00080000:2.0:1787827523.970523:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827523.970526:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:2.0:1787827523.970536:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787827523.970549:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8f480 x1874672572466048/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787827523.970562:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1787827523.970564:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8cb40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572465920:192.168.203.129@tcp:250:kworker.0 00000100:00080000:2.0:1787827523.970569:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1787827523.970572:0:37:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827523.970577:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 476 00000100:00080000:2.0:1787827523.970580:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827523.970584:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:2.0:1787827523.970598:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787827523.970611:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8c000 x1874672572466176/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1787827523.970623:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787827523.970662:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827523/real 1787827523] req@ffff8cb998a8cb40 x1874672572465920/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827548 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827523.970676:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8cb40 x1874672572465920/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827548 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827523.970685:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827523.970688:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787827523.970691:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8cb40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572465920:192.168.203.129@tcp:250:kworker.0 00000100:00100000:0.0:1787827523.970711:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572466048:192.168.203.129@tcp:38:kworker.0 00000800:00000100:0.0:1787827523.970729:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787827523.970738:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827523/real 1787827523] req@ffff8cb998a8f480 x1874672572466048/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827548 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827523.970758:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8f480 x1874672572466048/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827548 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827523.970764:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827523.970767:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787827523.970770:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572466048:192.168.203.129@tcp:38:kworker.0 00000100:00100000:0.0:1787827523.970776:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572466176:192.168.203.129@tcp:38:kworker.0 00000800:00000100:0.0:1787827523.970791:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787827523.970801:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827523/real 1787827523] req@ffff8cb998a8c000 x1874672572466176/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827548 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827523.970811:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8c000 x1874672572466176/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827548 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827523.970817:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827523.970820:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787827523.970823:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572466176:192.168.203.129@tcp:38:kworker.0 00010000:00010000:1.0:1787827524.802636:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787827525.826683:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787827526.850467:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787827527.874427:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787827528.898449:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787827529.090392:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787827529.090400:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787827529.090403:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787827529.090410:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 481 00000100:00080000:1.0:1787827529.090418:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787827529.090424:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:1.0:1787827529.090474:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f3b480 x1874672572466304/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787827529.090498:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787827529.090502:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787827529.090504:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787827529.090507:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 481 00000100:00080000:1.0:1787827529.090511:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787827529.090514:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:1.0:1787827529.090525:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787827529.090535:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f392c0 x1874672572466432/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787827529.090545:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f3b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572466304:192.168.203.129@tcp:250:kworker.0 00000100:00080000:1.0:1787827529.090550:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787827529.090554:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1787827529.090556:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787827529.090560:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 481 00000100:00080000:1.0:1787827529.090562:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787827529.090595:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:1.0:1787827529.090600:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787827529.090609:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f3b0c0 x1874672572466560/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1787827529.090638:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787827529.090694:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827529/real 1787827529] req@ffff8cb999f3b480 x1874672572466304/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827554 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827529.090708:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f3b480 x1874672572466304/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827554 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827529.090721:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827529.090724:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787827529.090727:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f3b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572466304:192.168.203.129@tcp:250:kworker.0 00000100:00100000:0.0:1787827529.090748:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f392c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572466432:192.168.203.129@tcp:38:kworker.0 00000800:00000100:0.0:1787827529.090766:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787827529.090775:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827529/real 1787827529] req@ffff8cb999f392c0 x1874672572466432/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827554 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827529.090806:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f392c0 x1874672572466432/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827554 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827529.090815:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827529.090819:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787827529.090822:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f392c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572466432:192.168.203.129@tcp:38:kworker.0 00000100:00100000:0.0:1787827529.090829:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f3b0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572466560:192.168.203.129@tcp:38:kworker.0 00000800:00000100:0.0:1787827529.090894:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787827529.090902:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827529/real 1787827529] req@ffff8cb999f3b0c0 x1874672572466560/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827554 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827529.090910:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f3b0c0 x1874672572466560/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827554 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827529.090915:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827529.090918:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787827529.090937:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f3b0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572466560:192.168.203.129@tcp:38:kworker.0 00010000:00010000:2.0:1787827529.922473:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787827530.946414:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787827531.970475:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787827532.994405:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787827534.018465:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787827534.210390:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827534.210399:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787827534.210403:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827534.210412:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 486 00000100:00080000:2.0:1787827534.210422:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827534.210428:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:2.0:1787827534.210463:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8e580 x1874672572466688/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787827534.210488:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827534.210491:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787827534.210493:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827534.210496:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 486 00000100:00080000:2.0:1787827534.210499:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827534.210501:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:2.0:1787827534.210510:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787827534.210521:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8c780 x1874672572466816/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787827534.210534:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827534.210554:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1787827534.210557:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827534.210564:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 486 00000100:00080000:2.0:1787827534.210571:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1787827534.210580:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8e580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572466688:192.168.203.129@tcp:250:kworker.0 00000100:00080000:2.0:1787827534.210581:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:2.0:1787827534.210601:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787827534.210622:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8f0c0 x1874672572466944/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1787827534.210724:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787827534.210784:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827534/real 1787827534] req@ffff8cb998a8e580 x1874672572466688/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827559 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827534.210830:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8e580 x1874672572466688/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827559 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827534.210863:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827534.210871:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787827534.210878:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8e580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572466688:192.168.203.129@tcp:250:kworker.0 00000100:00100000:0.0:1787827534.210893:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8c780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572466816:192.168.203.129@tcp:38:kworker.0 00000800:00000100:0.0:1787827534.210936:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787827534.210952:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827534/real 1787827534] req@ffff8cb998a8c780 x1874672572466816/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827559 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827534.210968:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8c780 x1874672572466816/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827559 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827534.210983:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827534.210989:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787827534.211006:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8c780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572466816:192.168.203.129@tcp:38:kworker.0 00000100:00100000:0.0:1787827534.211056:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8f0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572466944:192.168.203.129@tcp:38:kworker.0 00000800:00000100:0.0:1787827534.211100:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787827534.211114:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827534/real 1787827534] req@ffff8cb998a8f0c0 x1874672572466944/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827559 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827534.211127:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8f0c0 x1874672572466944/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827559 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827534.211136:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827534.211140:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787827534.211144:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8f0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572466944:192.168.203.129@tcp:38:kworker.0 00010000:00010000:2.0:1787827535.042399:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787827536.066512:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787827537.090430:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787827538.114489:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787827539.138496:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787827539.330427:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787827539.330438:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787827539.330444:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787827539.330455:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 491 00000100:00080000:0.0:1787827539.330467:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787827539.330475:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:0.0:1787827539.330521:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c68000 x1874672572467072/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787827539.330555:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787827539.330560:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787827539.330562:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787827539.330566:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 491 00000100:00080000:0.0:1787827539.330570:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787827539.330573:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:0.0:1787827539.330602:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787827539.330619:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c69680 x1874672572467200/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787827539.330651:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787827539.330655:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1787827539.330658:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787827539.330663:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 491 00000100:00080000:0.0:1787827539.330669:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1787827539.330680:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c68000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572467072:192.168.203.129@tcp:250:kworker.0 00000100:00080000:0.0:1787827539.330681:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:0.0:1787827539.330697:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787827539.330714:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c6b480 x1874672572467328/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1787827539.330836:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:3.0:1787827539.330896:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827539/real 1787827539] req@ffff8cb999c68000 x1874672572467072/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827564 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827539.330947:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c68000 x1874672572467072/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827564 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827539.330968:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827539.330977:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:3.0:1787827539.330986:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c68000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572467072:192.168.203.129@tcp:250:kworker.0 00000100:00100000:3.0:1787827539.331001:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c69680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572467200:192.168.203.129@tcp:38:kworker.0 00000800:00000100:3.0:1787827539.331064:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:3.0:1787827539.331087:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827539/real 1787827539] req@ffff8cb999c69680 x1874672572467200/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827564 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827539.331111:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c69680 x1874672572467200/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827564 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827539.331126:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827539.331131:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:3.0:1787827539.331156:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c69680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572467200:192.168.203.129@tcp:38:kworker.0 00000100:00100000:3.0:1787827539.331203:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c6b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572467328:192.168.203.129@tcp:38:kworker.0 00000800:00000100:3.0:1787827539.331245:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:3.0:1787827539.331288:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827539/real 1787827539] req@ffff8cb999c6b480 x1874672572467328/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827564 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827539.331373:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c6b480 x1874672572467328/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827564 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827539.331390:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827539.331396:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:3.0:1787827539.331403:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c6b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572467328:192.168.203.129@tcp:38:kworker.0 00010000:00010000:0.0:1787827540.162418:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787827541.186379:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787827542.210420:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787827543.234377:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787827544.258399:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787827544.450345:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787827544.450355:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787827544.450360:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787827544.450369:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 496 00000100:00080000:0.0:1787827544.450380:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787827544.450386:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:0.0:1787827544.450420:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c6bc00 x1874672572467456/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787827544.450448:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787827544.450452:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787827544.450467:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787827544.450473:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 496 00000100:00080000:0.0:1787827544.450477:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787827544.450481:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:3.0:1787827544.450489:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c6bc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572467456:192.168.203.129@tcp:250:kworker.0 00000008:00100000:0.0:1787827544.450490:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787827544.450505:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c68f00 x1874672572467584/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787827544.450515:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787827544.450517:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1787827544.450519:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787827544.450522:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 496 00000100:00080000:0.0:1787827544.450525:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787827544.450528:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:0.0:1787827544.450531:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787827544.450539:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c69e00 x1874672572467712/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1787827544.450552:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:3.0:1787827544.450589:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827544/real 1787827544] req@ffff8cb999c6bc00 x1874672572467456/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827569 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827544.450602:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c6bc00 x1874672572467456/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827569 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827544.450610:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827544.450613:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:3.0:1787827544.450616:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c6bc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572467456:192.168.203.129@tcp:250:kworker.0 00000100:00100000:3.0:1787827544.450635:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c68f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572467584:192.168.203.129@tcp:38:kworker.0 00000800:00000100:3.0:1787827544.450651:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:3.0:1787827544.450659:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827544/real 1787827544] req@ffff8cb999c68f00 x1874672572467584/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827569 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827544.450667:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c68f00 x1874672572467584/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827569 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827544.450684:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827544.450686:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:3.0:1787827544.450689:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c68f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572467584:192.168.203.129@tcp:38:kworker.0 00000100:00100000:3.0:1787827544.450693:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c69e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572467712:192.168.203.129@tcp:38:kworker.0 00000800:00000100:3.0:1787827544.450707:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:3.0:1787827544.450714:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827544/real 1787827544] req@ffff8cb999c69e00 x1874672572467712/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827569 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827544.450722:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c69e00 x1874672572467712/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827569 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827544.450744:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827544.450747:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:3.0:1787827544.450749:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c69e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572467712:192.168.203.129@tcp:38:kworker.0 00010000:00010000:2.0:1787827545.282416:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787827546.306459:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787827547.330391:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787827548.354398:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787827549.378448:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787827549.570327:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787827549.570335:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787827549.570338:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787827549.570346:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 501 00000100:00080000:0.0:1787827549.570355:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787827549.570361:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:0.0:1787827549.570409:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c6b840 x1874672572467840/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787827549.570430:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787827549.570434:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787827549.570436:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787827549.570440:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 501 00000100:00080000:0.0:1787827549.570444:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787827549.570449:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:0.0:1787827549.570459:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787827549.570460:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c6b840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572467840:192.168.203.129@tcp:250:kworker.0 00000100:00100000:0.0:1787827549.570471:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c69a40 x1874672572467968/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787827549.570481:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787827549.570484:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1787827549.570486:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787827549.570498:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 501 00000100:00080000:0.0:1787827549.570504:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787827549.570507:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:0.0:1787827549.570512:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000800:00000100:3.0:1787827549.570518:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787827549.570523:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c69e00 x1874672572468096/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1787827549.570556:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827549/real 1787827549] req@ffff8cb999c6b840 x1874672572467840/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827574 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827549.570574:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c6b840 x1874672572467840/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827574 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827549.570582:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827549.570584:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:3.0:1787827549.570593:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c6b840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572467840:192.168.203.129@tcp:250:kworker.0 00000100:00100000:3.0:1787827549.570612:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c69a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572467968:192.168.203.129@tcp:38:kworker.0 00000800:00000100:3.0:1787827549.570631:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:3.0:1787827549.570641:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827549/real 1787827549] req@ffff8cb999c69a40 x1874672572467968/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827574 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827549.570650:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c69a40 x1874672572467968/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827574 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827549.570655:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827549.570657:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:3.0:1787827549.570660:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c69a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572467968:192.168.203.129@tcp:38:kworker.0 00000100:00100000:3.0:1787827549.570663:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c69e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572468096:192.168.203.129@tcp:38:kworker.0 00000800:00000100:3.0:1787827549.570674:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:3.0:1787827549.570680:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827549/real 1787827549] req@ffff8cb999c69e00 x1874672572468096/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827574 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827549.570687:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c69e00 x1874672572468096/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827574 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827549.570706:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827549.570708:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:3.0:1787827549.570711:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c69e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572468096:192.168.203.129@tcp:38:kworker.0 00010000:00010000:2.0:1787827550.402458:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787827551.426422:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787827552.450456:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787827553.474428:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787827554.498392:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787827554.690387:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827554.690397:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787827554.690401:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827554.690411:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 506 00000100:00080000:2.0:1787827554.690428:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827554.690436:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:2.0:1787827554.690522:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8cf00 x1874672572468224/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787827554.690561:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827554.690564:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787827554.690566:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827554.690569:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 506 00000100:00080000:2.0:1787827554.690572:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827554.690575:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:2.0:1787827554.690585:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787827554.690605:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8d680 x1874672572468352/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787827554.690614:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827554.690617:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1787827554.690619:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827554.690623:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 506 00000100:00080000:2.0:1787827554.690627:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1787827554.690628:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8cf00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572468224:192.168.203.129@tcp:250:kworker.0 00000100:00080000:2.0:1787827554.690630:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:2.0:1787827554.690634:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787827554.690645:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8e1c0 x1874672572468480/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1787827554.690732:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:3.0:1787827554.690805:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827554/real 1787827554] req@ffff8cb998a8cf00 x1874672572468224/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827579 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827554.690827:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8cf00 x1874672572468224/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827579 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827554.690840:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827554.690845:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:3.0:1787827554.690850:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8cf00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572468224:192.168.203.129@tcp:250:kworker.0 00000100:00100000:3.0:1787827554.690881:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8d680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572468352:192.168.203.129@tcp:38:kworker.0 00000800:00000100:3.0:1787827554.690908:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:3.0:1787827554.690923:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827554/real 1787827554] req@ffff8cb998a8d680 x1874672572468352/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827579 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827554.690937:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8d680 x1874672572468352/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827579 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827554.690946:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827554.690950:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:3.0:1787827554.690960:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8d680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572468352:192.168.203.129@tcp:38:kworker.0 00000100:00100000:3.0:1787827554.690968:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8e1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572468480:192.168.203.129@tcp:38:kworker.0 00000800:00000100:3.0:1787827554.690988:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:3.0:1787827554.690998:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827554/real 1787827554] req@ffff8cb998a8e1c0 x1874672572468480/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827579 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827554.691007:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8e1c0 x1874672572468480/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827579 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827554.691022:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827554.691025:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:3.0:1787827554.691027:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8e1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572468480:192.168.203.129@tcp:38:kworker.0 00010000:00010000:0.0:1787827555.522484:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787827556.546410:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787827557.570401:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787827558.594425:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787827559.618421:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787827559.810334:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827559.810358:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787827559.810361:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827559.810370:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 511 00000100:00080000:2.0:1787827559.810379:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827559.810385:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:2.0:1787827559.810414:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8da40 x1874672572468608/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787827559.810437:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827559.810440:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787827559.810441:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827559.810444:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 511 00000100:00080000:2.0:1787827559.810446:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827559.810448:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:2.0:1787827559.810467:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787827559.810476:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8fc00 x1874672572468736/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787827559.810484:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827559.810487:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1787827559.810488:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827559.810493:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 511 00000100:00100000:3.0:1787827559.810495:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8da40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572468608:192.168.203.129@tcp:250:kworker.0 00000100:00080000:2.0:1787827559.810496:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827559.810503:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:2.0:1787827559.810507:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787827559.810515:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8c3c0 x1874672572468864/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1787827559.810557:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:3.0:1787827559.810604:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827559/real 1787827559] req@ffff8cb998a8da40 x1874672572468608/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827584 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827559.810632:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8da40 x1874672572468608/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827584 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827559.810642:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827559.810646:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:3.0:1787827559.810650:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8da40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572468608:192.168.203.129@tcp:250:kworker.0 00000100:00100000:3.0:1787827559.810671:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8fc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572468736:192.168.203.129@tcp:38:kworker.0 00000800:00000100:3.0:1787827559.810696:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:3.0:1787827559.810709:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827559/real 1787827559] req@ffff8cb998a8fc00 x1874672572468736/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827584 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827559.810719:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8fc00 x1874672572468736/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827584 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827559.810728:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827559.810732:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:3.0:1787827559.810736:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8fc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572468736:192.168.203.129@tcp:38:kworker.0 00000100:00100000:3.0:1787827559.810752:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8c3c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572468864:192.168.203.129@tcp:38:kworker.0 00000800:00000100:3.0:1787827559.810779:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:3.0:1787827559.810856:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827559/real 1787827559] req@ffff8cb998a8c3c0 x1874672572468864/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827584 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827559.810870:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8c3c0 x1874672572468864/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827584 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827559.810876:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827559.810878:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:3.0:1787827559.810881:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8c3c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572468864:192.168.203.129@tcp:38:kworker.0 00010000:00010000:2.0:1787827560.642411:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787827561.666412:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787827562.690427:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787827563.714435:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787827564.738390:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787827564.930388:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787827564.930399:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787827564.930404:0:37:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787827564.930416:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 517 00000100:00080000:1.0:1787827564.930427:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787827564.930435:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:1.0:1787827564.930474:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f38b40 x1874672572468992/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787827564.930507:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787827564.930512:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787827564.930515:0:37:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787827564.930519:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 517 00000100:00080000:1.0:1787827564.930525:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787827564.930530:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:1.0:1787827564.930542:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787827564.930557:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f39e00 x1874672572469120/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1787827564.930568:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f38b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572468992:192.168.203.129@tcp:250:kworker.0 00000100:00080000:1.0:1787827564.930572:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787827564.930581:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1787827564.930630:0:37:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787827564.930636:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 517 00000100:00080000:1.0:1787827564.930642:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787827564.930647:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:1.0:1787827564.930653:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787827564.930667:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f38f00 x1874672572469248/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1787827564.930679:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:3.0:1787827564.930728:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827564/real 1787827564] req@ffff8cb999f38b40 x1874672572468992/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827589 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827564.930762:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f38b40 x1874672572468992/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827589 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827564.930775:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827564.930780:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:3.0:1787827564.930784:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f38b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572468992:192.168.203.129@tcp:250:kworker.0 00000100:00100000:3.0:1787827564.930810:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f39e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572469120:192.168.203.129@tcp:38:kworker.0 00000800:00000100:3.0:1787827564.930838:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:3.0:1787827564.930854:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827564/real 1787827564] req@ffff8cb999f39e00 x1874672572469120/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827589 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827564.930879:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f39e00 x1874672572469120/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827589 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827564.930891:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827564.930895:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:3.0:1787827564.930899:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f39e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572469120:192.168.203.129@tcp:38:kworker.0 00000100:00100000:3.0:1787827564.930907:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f38f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572469248:192.168.203.129@tcp:38:kworker.0 00000800:00000100:3.0:1787827564.930934:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:3.0:1787827564.930948:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827564/real 1787827564] req@ffff8cb999f38f00 x1874672572469248/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827589 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827564.930963:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f38f00 x1874672572469248/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827589 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827564.930973:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827564.930978:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:3.0:1787827564.930982:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f38f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572469248:192.168.203.129@tcp:38:kworker.0 00010000:00010000:2.0:1787827565.762426:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787827566.786394:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787827567.810457:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787827568.834435:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787827569.858496:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787827570.050348:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787827570.050358:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787827570.050364:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787827570.050375:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 522 00000100:00080000:0.0:1787827570.050386:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787827570.050394:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:0.0:1787827570.050431:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c68f00 x1874672572469376/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787827570.050458:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787827570.050462:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787827570.050465:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787827570.050469:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 522 00000100:00080000:0.0:1787827570.050473:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787827570.050478:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:0.0:1787827570.050489:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787827570.050493:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c68f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572469376:192.168.203.129@tcp:250:kworker.0 00000100:00100000:0.0:1787827570.050506:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c6bc00 x1874672572469504/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787827570.050521:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787827570.050525:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1787827570.050527:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787827570.050531:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 522 00000100:00080000:0.0:1787827570.050536:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787827570.050540:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000800:00000100:3.0:1787827570.050567:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000008:00100000:0.0:1787827570.050568:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787827570.050602:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c6b480 x1874672572469632/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1787827570.050624:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827570/real 1787827570] req@ffff8cb999c68f00 x1874672572469376/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827595 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827570.050641:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c68f00 x1874672572469376/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827595 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827570.050661:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827570.050665:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:3.0:1787827570.050668:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c68f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572469376:192.168.203.129@tcp:250:kworker.0 00000100:00100000:3.0:1787827570.050697:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c6bc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572469504:192.168.203.129@tcp:38:kworker.0 00000800:00000100:3.0:1787827570.050719:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:3.0:1787827570.050730:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827570/real 1787827570] req@ffff8cb999c6bc00 x1874672572469504/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827595 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827570.050739:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c6bc00 x1874672572469504/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827595 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827570.050746:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827570.050748:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:3.0:1787827570.050751:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c6bc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572469504:192.168.203.129@tcp:38:kworker.0 00000100:00100000:3.0:1787827570.050755:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c6b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572469632:192.168.203.129@tcp:38:kworker.0 00000800:00000100:3.0:1787827570.050778:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:3.0:1787827570.050786:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827570/real 1787827570] req@ffff8cb999c6b480 x1874672572469632/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827595 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827570.050795:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c6b480 x1874672572469632/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827595 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827570.050801:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827570.050803:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:3.0:1787827570.050806:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c6b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572469632:192.168.203.129@tcp:38:kworker.0 00010000:00010000:0.0:1787827570.882408:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787827571.906395:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787827572.930435:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787827573.954457:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787827574.978479:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787827575.170348:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787827575.170359:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787827575.170364:0:37:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787827575.170375:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 527 00000100:00080000:0.0:1787827575.170384:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787827575.170390:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:0.0:1787827575.170423:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c69680 x1874672572469760/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787827575.170448:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787827575.170452:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787827575.170453:0:37:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787827575.170456:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 527 00000100:00080000:0.0:1787827575.170459:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787827575.170461:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:0.0:1787827575.170469:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787827575.170479:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c68000 x1874672572469888/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787827575.170489:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1787827575.170491:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c69680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572469760:192.168.203.129@tcp:250:kworker.0 00000100:00080000:0.0:1787827575.170494:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1787827575.170528:0:37:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787827575.170532:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 527 00000100:00080000:0.0:1787827575.170536:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787827575.170540:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:0.0:1787827575.170551:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000800:00000100:3.0:1787827575.170570:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787827575.170574:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c6a1c0 x1874672572470016/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1787827575.170610:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827575/real 1787827575] req@ffff8cb999c69680 x1874672572469760/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827600 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827575.170629:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c69680 x1874672572469760/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827600 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827575.170641:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827575.170646:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:3.0:1787827575.170652:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c69680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572469760:192.168.203.129@tcp:250:kworker.0 00000100:00100000:3.0:1787827575.170676:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c68000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572469888:192.168.203.129@tcp:38:kworker.0 00000800:00000100:3.0:1787827575.170703:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:3.0:1787827575.170719:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827575/real 1787827575] req@ffff8cb999c68000 x1874672572469888/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827600 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827575.170734:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c68000 x1874672572469888/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827600 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827575.170744:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827575.170749:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:3.0:1787827575.170753:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c68000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572469888:192.168.203.129@tcp:38:kworker.0 00000100:00100000:3.0:1787827575.170760:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c6a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572470016:192.168.203.129@tcp:38:kworker.0 00000800:00000100:3.0:1787827575.170785:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:3.0:1787827575.170796:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827575/real 1787827575] req@ffff8cb999c6a1c0 x1874672572470016/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827600 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827575.170814:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c6a1c0 x1874672572470016/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827600 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827575.170824:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827575.170828:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:3.0:1787827575.170832:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c6a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572470016:192.168.203.129@tcp:38:kworker.0 00010000:00010000:2.0:1787827576.002480:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787827577.026435:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787827578.050444:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787827579.074480:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787827580.098445:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787827580.290379:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827580.290389:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787827580.290394:0:37:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827580.290403:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 532 00000100:00080000:2.0:1787827580.290413:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827580.290421:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:2.0:1787827580.290455:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8de00 x1874672572470144/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787827580.290492:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827580.290496:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787827580.290498:0:37:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827580.290502:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 532 00000100:00080000:2.0:1787827580.290506:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827580.290525:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:2.0:1787827580.290535:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787827580.290544:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8de00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572470144:192.168.203.129@tcp:250:kworker.0 00000100:00100000:2.0:1787827580.290552:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8d2c0 x1874672572470272/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787827580.290566:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827580.290570:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1787827580.290572:0:37:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827580.290576:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 532 00000100:00080000:2.0:1787827580.290591:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827580.290597:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:2.0:1787827580.290602:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787827580.290616:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8c3c0 x1874672572470400/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1787827580.290642:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8d2c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572470272:192.168.203.129@tcp:38:kworker.0 00000100:00100000:3.0:1787827580.290666:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8c3c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572470400:192.168.203.129@tcp:38:kworker.0 00010000:00010000:2.0:1787827581.122532:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787827582.146408:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787827583.170520:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:0.0:1787827583.362557:0:2384:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.129:988 00000400:00000100:0.0:1787827583.362604:0:2384:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.129@tcp at host 192.168.203.129 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:0.0:1787827583.362622:0:2384:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.129@tcp 00000800:00000100:0.0:1787827583.362670:0:2384:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.129@tcp 00000800:00000100:0.0:1787827583.362681:0:2384:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.129@tcp 00000100:00100000:3.0:1787827583.362744:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827580/real 1787827583] req@ffff8cb998a8d2c0 x1874672572470272/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827605 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827583.362777:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8d2c0 x1874672572470272/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827605 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827583.362798:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827583.362806:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:3.0:1787827583.362812:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8d2c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572470272:192.168.203.129@tcp:38:kworker.0 00000100:00100000:3.0:1787827583.362823:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827580/real 1787827583] req@ffff8cb998a8c3c0 x1874672572470400/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827605 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827583.362842:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8c3c0 x1874672572470400/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827605 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827583.362883:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827583.362888:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:3.0:1787827583.362892:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8c3c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572470400:192.168.203.129@tcp:38:kworker.0 00000100:00100000:3.0:1787827583.362900:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827580/real 1787827583] req@ffff8cb998a8de00 x1874672572470144/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827605 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827583.362916:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8de00 x1874672572470144/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827605 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827583.362924:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827583.362928:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:3.0:1787827583.362931:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8de00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572470144:192.168.203.129@tcp:250:kworker.0 00010000:00010000:0.0:1787827584.194719:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787827585.218699:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787827585.410376:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827585.410393:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787827585.410398:0:37:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827585.410414:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 537 00000100:00080000:2.0:1787827585.410430:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827585.410455:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:2.0:1787827585.410532:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8fc00 x1874672572470528/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787827585.410599:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827585.410604:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787827585.410607:0:37:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827585.410612:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 537 00000100:00080000:2.0:1787827585.410617:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827585.410624:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:2.0:1787827585.410639:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787827585.410682:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8da40 x1874672572470656/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787827585.410695:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827585.410701:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1787827585.410707:0:37:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1787827585.410709:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8fc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572470528:192.168.203.129@tcp:250:kworker.0 00000100:00080000:2.0:1787827585.410716:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 537 00000100:00080000:2.0:1787827585.410724:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827585.410740:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:2.0:1787827585.410746:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787827585.410765:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8e1c0 x1874672572470784/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1787827585.410819:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:3.0:1787827585.410904:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827585/real 1787827585] req@ffff8cb998a8fc00 x1874672572470528/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827610 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827585.410939:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8fc00 x1874672572470528/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827610 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827585.410954:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827585.410958:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:3.0:1787827585.410963:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8fc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572470528:192.168.203.129@tcp:250:kworker.0 00000100:00100000:3.0:1787827585.410993:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8da40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572470656:192.168.203.129@tcp:38:kworker.0 00000800:00000100:3.0:1787827585.411059:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:3.0:1787827585.411077:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827585/real 1787827585] req@ffff8cb998a8da40 x1874672572470656/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827610 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827585.411093:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8da40 x1874672572470656/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827610 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827585.411099:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827585.411102:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:3.0:1787827585.411105:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8da40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572470656:192.168.203.129@tcp:38:kworker.0 00000100:00100000:3.0:1787827585.411111:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8e1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572470784:192.168.203.129@tcp:38:kworker.0 00000800:00000100:3.0:1787827585.411136:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:3.0:1787827585.411149:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827585/real 1787827585] req@ffff8cb998a8e1c0 x1874672572470784/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827610 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827585.411163:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8e1c0 x1874672572470784/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827610 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827585.411172:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827585.411177:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:3.0:1787827585.411181:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8e1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572470784:192.168.203.129@tcp:38:kworker.0 00010000:00010000:0.0:1787827586.242740:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787827587.266469:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787827588.290770:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787827589.314699:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787827590.338770:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787827590.530537:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787827590.530554:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787827590.530562:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787827590.530579:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 542 00000100:00080000:0.0:1787827590.530595:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787827590.530604:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:0.0:1787827590.530661:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c692c0 x1874672572470912/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787827590.530701:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787827590.530728:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787827590.530731:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787827590.530737:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 542 00000100:00080000:0.0:1787827590.530742:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787827590.530750:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:0.0:1787827590.530769:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787827590.530803:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c6a580 x1874672572471040/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1787827590.530803:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c692c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572470912:192.168.203.129@tcp:250:kworker.0 00000100:00080000:0.0:1787827590.530831:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787827590.530834:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1787827590.530836:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787827590.530840:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 542 00000100:00080000:0.0:1787827590.530861:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787827590.530867:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:0.0:1787827590.530874:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787827590.530893:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c6a1c0 x1874672572471168/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1787827590.530922:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:3.0:1787827590.531006:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827590/real 1787827590] req@ffff8cb999c692c0 x1874672572470912/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827615 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827590.531068:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c692c0 x1874672572470912/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827615 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827590.531103:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827590.531111:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:3.0:1787827590.531118:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c692c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572470912:192.168.203.129@tcp:250:kworker.0 00000100:00100000:3.0:1787827590.531159:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c6a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572471040:192.168.203.129@tcp:38:kworker.0 00000800:00000100:3.0:1787827590.531200:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:3.0:1787827590.531236:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827590/real 1787827590] req@ffff8cb999c6a580 x1874672572471040/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827615 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827590.531250:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c6a580 x1874672572471040/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827615 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827590.531259:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827590.531263:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:3.0:1787827590.531267:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c6a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572471040:192.168.203.129@tcp:38:kworker.0 00000100:00100000:3.0:1787827590.531274:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c6a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572471168:192.168.203.129@tcp:38:kworker.0 00000800:00000100:3.0:1787827590.531367:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:3.0:1787827590.531388:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827590/real 1787827590] req@ffff8cb999c6a1c0 x1874672572471168/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827615 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827590.531435:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c6a1c0 x1874672572471168/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827615 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827590.531477:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827590.531484:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:3.0:1787827590.531491:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c6a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572471168:192.168.203.129@tcp:38:kworker.0 00010000:00010000:2.0:1787827591.363339:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787827592.386590:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787827593.410376:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787827594.434583:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787827595.458445:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787827595.650427:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787827595.650438:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787827595.650445:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787827595.650475:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 547 00000100:00080000:0.0:1787827595.650489:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787827595.650498:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:0.0:1787827595.650571:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c68000 x1874672572471296/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787827595.650608:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787827595.650614:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787827595.650616:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787827595.650622:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 547 00000100:00080000:0.0:1787827595.650628:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787827595.650635:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:0.0:1787827595.650646:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787827595.650666:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c68000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572471296:192.168.203.129@tcp:250:kworker.0 00000100:00100000:0.0:1787827595.650674:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c69680 x1874672572471424/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787827595.650704:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787827595.650707:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1787827595.650710:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787827595.650714:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 547 00000100:00080000:0.0:1787827595.650718:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787827595.650722:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:0.0:1787827595.650728:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787827595.650741:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c6b480 x1874672572471552/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1787827595.650757:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:3.0:1787827595.650804:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827595/real 1787827595] req@ffff8cb999c68000 x1874672572471296/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827620 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827595.650966:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c68000 x1874672572471296/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827620 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827595.650977:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827595.650981:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:3.0:1787827595.650984:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c68000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572471296:192.168.203.129@tcp:250:kworker.0 00000100:00100000:3.0:1787827595.651178:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c69680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572471424:192.168.203.129@tcp:38:kworker.0 00000800:00000100:3.0:1787827595.651366:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:3.0:1787827595.651421:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827595/real 1787827595] req@ffff8cb999c69680 x1874672572471424/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827620 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827595.651446:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c69680 x1874672572471424/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827620 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827595.651459:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827595.651463:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:3.0:1787827595.651467:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c69680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572471424:192.168.203.129@tcp:38:kworker.0 00000100:00100000:3.0:1787827595.651486:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c6b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572471552:192.168.203.129@tcp:38:kworker.0 00000800:00000100:3.0:1787827595.651542:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:3.0:1787827595.651554:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827595/real 1787827595] req@ffff8cb999c6b480 x1874672572471552/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827620 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827595.651564:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c6b480 x1874672572471552/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827620 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827595.651571:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827595.651573:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:3.0:1787827595.651576:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c6b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572471552:192.168.203.129@tcp:38:kworker.0 00010000:00010000:2.0:1787827596.482477:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787827597.506603:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787827598.530509:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787827599.554405:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787827600.578507:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787827600.770497:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787827600.770510:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787827600.770558:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787827600.770574:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 552 00000100:00080000:0.0:1787827600.770602:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787827600.770612:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:0.0:1787827600.770679:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c6bc00 x1874672572471680/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787827600.770717:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787827600.770721:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787827600.770724:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787827600.770729:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 552 00000100:00080000:0.0:1787827600.770733:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787827600.770737:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:0.0:1787827600.770752:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787827600.770770:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c68f00 x1874672572471808/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1787827600.770806:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c6bc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572471680:192.168.203.129@tcp:250:kworker.0 00000100:00080000:0.0:1787827600.770812:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787827600.770822:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1787827600.770826:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787827600.770845:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 552 00000100:00080000:0.0:1787827600.770853:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787827600.770861:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:0.0:1787827600.770871:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787827600.770892:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c69e00 x1874672572471936/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1787827600.770920:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:3.0:1787827600.771002:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827600/real 1787827600] req@ffff8cb999c6bc00 x1874672572471680/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827625 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827600.771048:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c6bc00 x1874672572471680/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827625 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827600.771070:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827600.771079:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:3.0:1787827600.771086:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c6bc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572471680:192.168.203.129@tcp:250:kworker.0 00000100:00100000:3.0:1787827600.771147:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c68f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572471808:192.168.203.129@tcp:38:kworker.0 00000800:00000100:3.0:1787827600.771190:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:3.0:1787827600.771223:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827600/real 1787827600] req@ffff8cb999c68f00 x1874672572471808/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827625 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827600.771314:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c68f00 x1874672572471808/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827625 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827600.771326:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827600.771330:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:3.0:1787827600.771360:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c68f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572471808:192.168.203.129@tcp:38:kworker.0 00000100:00100000:3.0:1787827600.771376:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c69e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572471936:192.168.203.129@tcp:38:kworker.0 00000800:00000100:3.0:1787827600.771410:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:3.0:1787827600.771426:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827600/real 1787827600] req@ffff8cb999c69e00 x1874672572471936/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827625 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827600.771446:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c69e00 x1874672572471936/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827625 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827600.771456:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827600.771460:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:3.0:1787827600.771464:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c69e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572471936:192.168.203.129@tcp:38:kworker.0 00010000:00010000:0.0:1787827601.602528:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787827602.626461:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787827603.650455:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787827604.674488:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787827605.698416:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787827605.890379:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787827605.890387:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787827605.890390:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787827605.890398:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 558 00000100:00080000:1.0:1787827605.890408:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787827605.890413:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:1.0:1787827605.890445:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f3b840 x1874672572472064/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787827605.890470:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787827605.890473:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787827605.890474:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787827605.890476:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 558 00000100:00080000:1.0:1787827605.890479:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787827605.890481:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:1.0:1787827605.890488:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787827605.890511:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f39a40 x1874672572472192/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787827605.890521:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1787827605.890522:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f3b840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572472064:192.168.203.129@tcp:250:kworker.0 00000100:00080000:1.0:1787827605.890524:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1787827605.890528:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787827605.890535:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 558 00000100:00080000:1.0:1787827605.890538:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787827605.890541:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:1.0:1787827605.890544:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787827605.890554:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f38f00 x1874672572472320/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1787827605.890587:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:3.0:1787827605.890633:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827605/real 1787827605] req@ffff8cb999f3b840 x1874672572472064/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827630 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827605.890652:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f3b840 x1874672572472064/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827630 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827605.890676:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827605.890681:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:3.0:1787827605.890687:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f3b840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572472064:192.168.203.129@tcp:250:kworker.0 00000100:00100000:3.0:1787827605.890709:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f39a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572472192:192.168.203.129@tcp:38:kworker.0 00000800:00000100:3.0:1787827605.890729:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:3.0:1787827605.890739:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827605/real 1787827605] req@ffff8cb999f39a40 x1874672572472192/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827630 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827605.890747:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f39a40 x1874672572472192/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827630 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827605.890752:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827605.890755:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:3.0:1787827605.890757:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f39a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572472192:192.168.203.129@tcp:38:kworker.0 00000100:00100000:3.0:1787827605.890761:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f38f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572472320:192.168.203.129@tcp:38:kworker.0 00000800:00000100:3.0:1787827605.890771:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:3.0:1787827605.890777:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827605/real 1787827605] req@ffff8cb999f38f00 x1874672572472320/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827630 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827605.890784:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f38f00 x1874672572472320/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827630 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827605.890790:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827605.890792:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:3.0:1787827605.890794:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f38f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572472320:192.168.203.129@tcp:38:kworker.0 00010000:00010000:2.0:1787827606.722406:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787827607.746480:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787827608.770476:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787827609.794556:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787827610.818544:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787827611.010387:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787827611.010397:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787827611.010401:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787827611.010409:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 563 00000100:00080000:1.0:1787827611.010419:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787827611.010425:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:1.0:1787827611.010470:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f39e00 x1874672572472448/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787827611.010495:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787827611.010498:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787827611.010499:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787827611.010502:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 563 00000100:00080000:1.0:1787827611.010505:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787827611.010519:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:1.0:1787827611.010529:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787827611.010529:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f39e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572472448:192.168.203.129@tcp:250:kworker.0 00000100:00100000:1.0:1787827611.010545:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f38b40 x1874672572472576/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787827611.010555:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787827611.010558:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1787827611.010560:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787827611.010564:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 563 00000100:00080000:1.0:1787827611.010567:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787827611.010570:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:1.0:1787827611.010573:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787827611.010585:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f3b0c0 x1874672572472704/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1787827611.010595:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:3.0:1787827611.010636:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827611/real 1787827611] req@ffff8cb999f39e00 x1874672572472448/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827636 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827611.010663:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f39e00 x1874672572472448/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827636 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827611.010675:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827611.010680:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:3.0:1787827611.010684:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f39e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572472448:192.168.203.129@tcp:250:kworker.0 00000100:00100000:3.0:1787827611.010780:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f38b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572472576:192.168.203.129@tcp:38:kworker.0 00000800:00000100:3.0:1787827611.010821:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:3.0:1787827611.010841:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827611/real 1787827611] req@ffff8cb999f38b40 x1874672572472576/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827636 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827611.010871:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f38b40 x1874672572472576/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827636 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827611.010881:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827611.010904:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:3.0:1787827611.010909:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f38b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572472576:192.168.203.129@tcp:38:kworker.0 00000100:00100000:3.0:1787827611.010917:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f3b0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572472704:192.168.203.129@tcp:38:kworker.0 00000800:00000100:3.0:1787827611.010942:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:3.0:1787827611.010956:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827611/real 1787827611] req@ffff8cb999f3b0c0 x1874672572472704/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827636 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827611.010995:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f3b0c0 x1874672572472704/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827636 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827611.011004:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827611.011008:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:3.0:1787827611.011011:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f3b0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572472704:192.168.203.129@tcp:38:kworker.0 00010000:00010000:2.0:1787827611.842550:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787827612.866442:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787827613.890589:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787827614.914498:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787827615.938507:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787827616.130448:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827616.130465:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787827616.130472:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827616.130488:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 568 00000100:00080000:3.0:1787827616.130505:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827616.130519:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:3.0:1787827616.130591:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb992317480 x1874672572472832/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787827616.130643:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827616.130651:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787827616.130655:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827616.130662:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 568 00000100:00080000:3.0:1787827616.130669:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827616.130677:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:3.0:1787827616.130694:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787827616.130721:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb992314f00 x1874672572472960/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787827616.130738:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb992317480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572472832:192.168.203.129@tcp:250:kworker.0 00000100:00080000:3.0:1787827616.130790:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827616.130798:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1787827616.130803:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827616.130812:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 568 00000100:00080000:3.0:1787827616.130822:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827616.130832:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:3.0:1787827616.130844:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000800:00000100:0.0:1787827616.130866:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:3.0:1787827616.130876:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb992315e00 x1874672572473088/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787827616.130934:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827616/real 1787827616] req@ffff8cb992317480 x1874672572472832/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827641 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827616.130962:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb992317480 x1874672572472832/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827641 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827616.130976:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827616.130981:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787827616.130987:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb992317480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572472832:192.168.203.129@tcp:250:kworker.0 00000100:00100000:0.0:1787827616.131019:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb992314f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572472960:192.168.203.129@tcp:38:kworker.0 00000800:00000100:0.0:1787827616.131050:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787827616.131066:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827616/real 1787827616] req@ffff8cb992314f00 x1874672572472960/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827641 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827616.131081:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb992314f00 x1874672572472960/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827641 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827616.131092:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827616.131096:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787827616.131114:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb992314f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572472960:192.168.203.129@tcp:38:kworker.0 00000100:00100000:0.0:1787827616.131122:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb992315e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572473088:192.168.203.129@tcp:38:kworker.0 00000800:00000100:0.0:1787827616.131150:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787827616.131164:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827616/real 1787827616] req@ffff8cb992315e00 x1874672572473088/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827641 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827616.131179:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb992315e00 x1874672572473088/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827641 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827616.131189:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827616.131193:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787827616.131197:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb992315e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572473088:192.168.203.129@tcp:38:kworker.0 00010000:00010000:0.0:1787827616.962465:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787827617.986448:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787827619.010448:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787827620.034379:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787827621.058362:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787827621.250371:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787827621.250382:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787827621.250387:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787827621.250396:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 573 00000100:00080000:1.0:1787827621.250405:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787827621.250411:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:1.0:1787827621.250441:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f392c0 x1874672572473216/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787827621.250465:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787827621.250467:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787827621.250482:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787827621.250485:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 573 00000100:00080000:1.0:1787827621.250488:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787827621.250491:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:1.0:1787827621.250499:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787827621.250507:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f3b480 x1874672572473344/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787827621.250515:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787827621.250517:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1787827621.250519:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787827621.250522:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 573 00000100:00080000:1.0:1787827621.250524:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787827621.250527:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:1.0:1787827621.250530:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787827621.250540:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f392c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572473216:192.168.203.129@tcp:250:kworker.0 00000100:00100000:1.0:1787827621.250540:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f3a580 x1874672572473472/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1787827621.250611:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787827621.250656:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827621/real 1787827621] req@ffff8cb999f392c0 x1874672572473216/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827646 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827621.250669:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f392c0 x1874672572473216/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827646 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827621.250677:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827621.250680:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787827621.250683:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f392c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572473216:192.168.203.129@tcp:250:kworker.0 00000100:00100000:0.0:1787827621.250687:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f3b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572473344:192.168.203.129@tcp:38:kworker.0 00000800:00000100:0.0:1787827621.250709:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787827621.250717:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827621/real 1787827621] req@ffff8cb999f3b480 x1874672572473344/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827646 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827621.250725:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f3b480 x1874672572473344/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827646 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827621.250730:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827621.250732:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787827621.250734:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f3b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572473344:192.168.203.129@tcp:38:kworker.0 00000100:00100000:0.0:1787827621.250754:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f3a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572473472:192.168.203.129@tcp:38:kworker.0 00000800:00000100:0.0:1787827621.250765:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787827621.250771:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827621/real 1787827621] req@ffff8cb999f3a580 x1874672572473472/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827646 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827621.250778:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f3a580 x1874672572473472/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827646 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827621.250783:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827621.250785:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787827621.250787:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f3a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572473472:192.168.203.129@tcp:38:kworker.0 00010000:00010000:2.0:1787827622.082415:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787827623.106420:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787827624.130446:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787827625.154487:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787827626.178443:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787827626.370428:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827626.370441:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787827626.370446:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827626.370459:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 578 00000100:00080000:3.0:1787827626.370472:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827626.370482:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:3.0:1787827626.370530:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb992316940 x1874672572473600/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787827626.370574:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827626.370580:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787827626.370582:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827626.370587:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 578 00000100:00080000:3.0:1787827626.370591:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827626.370595:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:3.0:1787827626.370630:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787827626.370649:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb9923170c0 x1874672572473728/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787827626.370667:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1787827626.370674:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb992316940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572473600:192.168.203.129@tcp:250:kworker.0 00000100:00080000:3.0:1787827626.370676:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1787827626.370686:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827626.370701:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 578 00000100:00080000:3.0:1787827626.370724:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827626.370730:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:3.0:1787827626.370738:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787827626.370759:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb9923161c0 x1874672572473856/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1787827626.370788:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787827626.370867:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827626/real 1787827626] req@ffff8cb992316940 x1874672572473600/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827651 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827626.370905:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb992316940 x1874672572473600/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827651 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827626.370937:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827626.370942:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787827626.370946:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb992316940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572473600:192.168.203.129@tcp:250:kworker.0 00000100:00100000:0.0:1787827626.370974:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb9923170c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572473728:192.168.203.129@tcp:38:kworker.0 00000800:00000100:0.0:1787827626.371026:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787827626.371086:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827626/real 1787827626] req@ffff8cb9923170c0 x1874672572473728/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827651 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827626.371129:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb9923170c0 x1874672572473728/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827651 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827626.371140:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827626.371143:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787827626.371146:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb9923170c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572473728:192.168.203.129@tcp:38:kworker.0 00000100:00100000:0.0:1787827626.371153:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb9923161c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572473856:192.168.203.129@tcp:38:kworker.0 00000800:00000100:0.0:1787827626.371196:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787827626.371229:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827626/real 1787827626] req@ffff8cb9923161c0 x1874672572473856/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827651 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827626.371245:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb9923161c0 x1874672572473856/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827651 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827626.371362:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827626.371368:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787827626.371373:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb9923161c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572473856:192.168.203.129@tcp:38:kworker.0 00010000:00010000:2.0:1787827627.202553:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787827628.226412:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787827629.250455:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787827630.274485:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787827631.298525:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787827631.490374:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827631.490390:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787827631.490396:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827631.490411:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 583 00000100:00080000:3.0:1787827631.490427:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827631.490440:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:3.0:1787827631.490489:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb992315a40 x1874672572473984/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787827631.490536:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827631.490544:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787827631.490548:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827631.490556:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 583 00000100:00080000:3.0:1787827631.490581:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827631.490592:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:0.0:1787827631.490609:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb992315a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572473984:192.168.203.129@tcp:250:kworker.0 00000800:00000100:0.0:1787827631.490683:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000008:00100000:3.0:1787827631.490700:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787827631.490736:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb9923161c0 x1874672572474112/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787827631.490738:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827631/real 1787827631] req@ffff8cb992315a40 x1874672572473984/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827656 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827631.490769:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb992315a40 x1874672572473984/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827656 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827631.490787:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827631.490794:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787827631.490801:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb992315a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572473984:192.168.203.129@tcp:250:kworker.0 00000100:00080000:3.0:1787827631.490960:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827631.490969:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1787827631.490974:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1787827631.491016:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb9923161c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572474112:192.168.203.129@tcp:38:kworker.0 00000800:00000100:0.0:1787827631.491059:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00080000:3.0:1787827631.491073:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 583 00000100:00100000:0.0:1787827631.491081:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827631/real 1787827631] req@ffff8cb9923161c0 x1874672572474112/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827656 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827631.491086:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827631.491104:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:0.0:1787827631.491105:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb9923161c0 x1874672572474112/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827656 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827631.491142:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827631.491147:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000008:00100000:3.0:1787827631.491148:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787827631.491164:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb9923161c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572474112:192.168.203.129@tcp:38:kworker.0 00000100:00100000:3.0:1787827631.491185:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb9923170c0 x1874672572474240/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787827631.491231:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb9923170c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572474240:192.168.203.129@tcp:38:kworker.0 00000800:00000100:0.0:1787827631.491306:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787827631.491344:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827631/real 1787827631] req@ffff8cb9923170c0 x1874672572474240/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827656 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827631.491369:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb9923170c0 x1874672572474240/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827656 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827631.491386:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827631.491392:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787827631.491399:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb9923170c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572474240:192.168.203.129@tcp:38:kworker.0 00010000:00010000:0.0:1787827632.322440:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787827633.346521:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787827634.370418:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787827635.394435:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787827636.418401:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787827636.610313:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827636.610321:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787827636.610324:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827636.610333:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 588 00000100:00080000:3.0:1787827636.610342:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827636.610348:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:3.0:1787827636.610378:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb992316940 x1874672572474368/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787827636.610410:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827636.610423:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787827636.610425:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827636.610429:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 588 00000100:00080000:3.0:1787827636.610432:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827636.610435:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:0.0:1787827636.610442:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb992316940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572474368:192.168.203.129@tcp:250:kworker.0 00000008:00100000:3.0:1787827636.610444:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787827636.610458:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb992315e00 x1874672572474496/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787827636.610466:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827636.610469:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1787827636.610470:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827636.610473:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 588 00000100:00080000:3.0:1787827636.610476:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827636.610478:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:3.0:1787827636.610481:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787827636.610488:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb992314f00 x1874672572474624/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1787827636.610501:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787827636.610534:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827636/real 1787827636] req@ffff8cb992316940 x1874672572474368/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827661 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827636.610547:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb992316940 x1874672572474368/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827661 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827636.610558:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827636.610562:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787827636.610566:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb992316940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572474368:192.168.203.129@tcp:250:kworker.0 00000100:00100000:0.0:1787827636.610590:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb992315e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572474496:192.168.203.129@tcp:38:kworker.0 00000800:00000100:0.0:1787827636.610619:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787827636.610629:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827636/real 1787827636] req@ffff8cb992315e00 x1874672572474496/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827661 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827636.610637:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb992315e00 x1874672572474496/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827661 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827636.610643:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827636.610645:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787827636.610648:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb992315e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572474496:192.168.203.129@tcp:38:kworker.0 00000100:00100000:0.0:1787827636.610652:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb992314f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572474624:192.168.203.129@tcp:38:kworker.0 00000800:00000100:0.0:1787827636.610663:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787827636.610680:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827636/real 1787827636] req@ffff8cb992314f00 x1874672572474624/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827661 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827636.610687:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb992314f00 x1874672572474624/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827661 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827636.610693:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827636.610694:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787827636.610696:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb992314f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572474624:192.168.203.129@tcp:38:kworker.0 00010000:00010000:0.0:1787827637.442422:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787827638.466379:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787827639.490385:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787827640.514493:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787827641.538412:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787827641.730330:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827641.730338:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787827641.730341:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827641.730350:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 593 00000100:00080000:3.0:1787827641.730359:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827641.730364:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:3.0:1787827641.730394:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb992317480 x1874672572474752/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787827641.730429:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827641.730432:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787827641.730434:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827641.730436:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 593 00000100:00080000:3.0:1787827641.730439:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827641.730442:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:3.0:1787827641.730449:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787827641.730459:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb992314b40 x1874672572474880/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787827641.730466:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827641.730468:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1787827641.730470:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827641.730472:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 593 00000100:00080000:3.0:1787827641.730474:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1787827641.730480:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb992317480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572474752:192.168.203.129@tcp:250:kworker.0 00000100:00080000:3.0:1787827641.730480:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:3.0:1787827641.730483:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787827641.730491:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb992314780 x1874672572475008/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1787827641.730562:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787827641.730600:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827641/real 1787827641] req@ffff8cb992317480 x1874672572474752/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827666 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827641.730621:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb992317480 x1874672572474752/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827666 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827641.730631:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827641.730635:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787827641.730638:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb992317480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572474752:192.168.203.129@tcp:250:kworker.0 00000100:00100000:0.0:1787827641.730647:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb992314b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572474880:192.168.203.129@tcp:38:kworker.0 00000800:00000100:0.0:1787827641.730667:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787827641.730680:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827641/real 1787827641] req@ffff8cb992314b40 x1874672572474880/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827666 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827641.730691:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb992314b40 x1874672572474880/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827666 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827641.730700:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827641.730703:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787827641.730706:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb992314b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572474880:192.168.203.129@tcp:38:kworker.0 00000100:00100000:0.0:1787827641.730734:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb992314780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572475008:192.168.203.129@tcp:38:kworker.0 00000800:00000100:0.0:1787827641.730754:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787827641.730766:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827641/real 1787827641] req@ffff8cb992314780 x1874672572475008/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827666 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827641.730792:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb992314780 x1874672572475008/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827666 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827641.730801:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827641.730804:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787827641.730808:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb992314780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572475008:192.168.203.129@tcp:38:kworker.0 00010000:00010000:2.0:1787827642.562410:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787827643.586407:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787827644.610416:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787827645.634373:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787827646.658515:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787827646.850400:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827646.850428:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787827646.850451:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827646.850463:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 599 00000100:00080000:2.0:1787827646.850476:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827646.850485:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:2.0:1787827646.850542:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8d680 x1874672572475136/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787827646.850586:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827646.850593:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787827646.850597:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827646.850603:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 599 00000100:00080000:2.0:1787827646.850611:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827646.850620:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:2.0:1787827646.850637:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787827646.850679:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8d680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572475136:192.168.203.129@tcp:250:kworker.0 00000100:00100000:2.0:1787827646.850707:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8cf00 x1874672572475264/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787827646.850726:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827646.850731:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1787827646.850734:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827646.850741:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 599 00000100:00080000:2.0:1787827646.850748:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827646.850754:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:2.0:1787827646.850761:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787827646.850808:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8f0c0 x1874672572475392/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787827646.850862:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8cf00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572475264:192.168.203.129@tcp:38:kworker.0 00000100:00100000:0.0:1787827646.850942:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8f0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572475392:192.168.203.129@tcp:38:kworker.0 00010000:00010000:0.0:1787827647.682429:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787827648.706393:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787827649.730430:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:1.0:1787827649.922476:0:2381:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.129:988 00000400:00000100:1.0:1787827649.922510:0:2381:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.129@tcp at host 192.168.203.129 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:1.0:1787827649.922524:0:2381:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.129@tcp 00000800:00000100:1.0:1787827649.922557:0:2381:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.129@tcp 00000800:00000100:1.0:1787827649.922576:0:2381:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.129@tcp 00000100:00100000:0.0:1787827649.922597:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827646/real 1787827649] req@ffff8cb998a8f0c0 x1874672572475392/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827671 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827649.922622:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8f0c0 x1874672572475392/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827671 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827649.922634:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827649.922639:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787827649.922644:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8f0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572475392:192.168.203.129@tcp:38:kworker.0 00000100:00100000:0.0:1787827649.922653:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827646/real 1787827649] req@ffff8cb998a8cf00 x1874672572475264/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827671 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827649.922682:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8cf00 x1874672572475264/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827671 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827649.922690:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827649.922694:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787827649.922698:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8cf00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572475264:192.168.203.129@tcp:38:kworker.0 00000100:00100000:0.0:1787827649.922704:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827646/real 1787827649] req@ffff8cb998a8d680 x1874672572475136/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827671 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827649.922716:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8d680 x1874672572475136/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827671 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827649.922725:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827649.922728:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787827649.922732:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8d680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572475136:192.168.203.129@tcp:250:kworker.0 00010000:00010000:0.0:1787827650.754792:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787827651.778630:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787827651.970428:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787827651.970445:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787827651.970450:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787827651.970475:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 604 00000100:00080000:1.0:1787827651.970487:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787827651.970494:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:1.0:1787827651.970539:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f383c0 x1874672572475520/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787827651.970577:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787827651.970581:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787827651.970583:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787827651.970588:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 604 00000100:00080000:1.0:1787827651.970593:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787827651.970598:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:1.0:1787827651.970611:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787827651.970626:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f3a1c0 x1874672572475648/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787827651.970640:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1787827651.970641:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f383c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572475520:192.168.203.129@tcp:250:kworker.0 00000100:00080000:1.0:1787827651.970650:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1787827651.970653:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787827651.970659:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 604 00000100:00080000:1.0:1787827651.970664:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787827651.970668:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:1.0:1787827651.970674:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787827651.970690:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f3a580 x1874672572475776/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1787827651.970746:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787827651.970798:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827651/real 1787827651] req@ffff8cb999f383c0 x1874672572475520/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827676 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827651.970827:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f383c0 x1874672572475520/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827676 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827651.970840:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827651.970845:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787827651.970859:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f383c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572475520:192.168.203.129@tcp:250:kworker.0 00000100:00100000:0.0:1787827651.970892:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f3a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572475648:192.168.203.129@tcp:38:kworker.0 00000800:00000100:0.0:1787827651.970923:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787827651.970937:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827651/real 1787827651] req@ffff8cb999f3a1c0 x1874672572475648/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827676 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827651.970950:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f3a1c0 x1874672572475648/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827676 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827651.970960:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827651.970963:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787827651.970966:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f3a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572475648:192.168.203.129@tcp:38:kworker.0 00000100:00100000:0.0:1787827651.970971:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f3a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572475776:192.168.203.129@tcp:38:kworker.0 00000800:00000100:0.0:1787827651.971004:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787827651.971013:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827651/real 1787827651] req@ffff8cb999f3a580 x1874672572475776/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827676 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827651.971021:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f3a580 x1874672572475776/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827676 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827651.971027:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827651.971029:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787827651.971032:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f3a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572475776:192.168.203.129@tcp:38:kworker.0 00010000:00010000:0.0:1787827652.802544:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787827653.826880:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787827654.850551:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787827655.874584:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787827656.898732:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787827657.090330:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787827657.090339:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787827657.090343:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787827657.090353:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 609 00000100:00080000:1.0:1787827657.090363:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787827657.090369:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:1.0:1787827657.090402:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f3b480 x1874672572475904/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787827657.090427:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787827657.090432:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787827657.090434:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787827657.090438:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 609 00000100:00080000:1.0:1787827657.090442:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787827657.090447:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:1.0:1787827657.090458:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787827657.090463:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f3b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572475904:192.168.203.129@tcp:250:kworker.0 00000100:00100000:1.0:1787827657.090474:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f392c0 x1874672572476032/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787827657.090486:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787827657.090489:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1787827657.090491:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787827657.090495:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 609 00000100:00080000:1.0:1787827657.090510:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000800:00000100:0.0:1787827657.090519:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00080000:1.0:1787827657.090526:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:1.0:1787827657.090532:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787827657.090546:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f3b0c0 x1874672572476160/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787827657.090555:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827657/real 1787827657] req@ffff8cb999f3b480 x1874672572475904/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827682 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827657.090571:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f3b480 x1874672572475904/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827682 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827657.090581:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827657.090584:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787827657.090587:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f3b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572475904:192.168.203.129@tcp:250:kworker.0 00000100:00100000:0.0:1787827657.090606:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f392c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572476032:192.168.203.129@tcp:38:kworker.0 00000800:00000100:0.0:1787827657.090622:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787827657.090636:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827657/real 1787827657] req@ffff8cb999f392c0 x1874672572476032/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827682 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827657.090649:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f392c0 x1874672572476032/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827682 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827657.090659:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827657.090663:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787827657.090667:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f392c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572476032:192.168.203.129@tcp:38:kworker.0 00000100:00100000:0.0:1787827657.090674:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f3b0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572476160:192.168.203.129@tcp:38:kworker.0 00000800:00000100:0.0:1787827657.090694:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787827657.090703:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827657/real 1787827657] req@ffff8cb999f3b0c0 x1874672572476160/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827682 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827657.090723:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f3b0c0 x1874672572476160/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827682 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827657.090737:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827657.090739:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787827657.090741:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f3b0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572476160:192.168.203.129@tcp:38:kworker.0 00010000:00010000:2.0:1787827657.923026:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787827658.946866:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787827659.970433:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787827660.994375:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787827662.018432:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787827662.210403:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827662.210410:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787827662.210414:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827662.210422:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 614 00000100:00080000:2.0:1787827662.210431:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827662.210436:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:2.0:1787827662.210466:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8c780 x1874672572476288/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787827662.210488:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827662.210491:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787827662.210492:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827662.210494:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 614 00000100:00080000:2.0:1787827662.210496:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827662.210499:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:2.0:1787827662.210506:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787827662.210513:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8e580 x1874672572476416/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787827662.210538:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827662.210541:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1787827662.210543:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827662.210546:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 614 00000100:00080000:2.0:1787827662.210550:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827662.210552:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:2.0:1787827662.210555:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787827662.210562:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8c000 x1874672572476544/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787827662.210590:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8c780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572476288:192.168.203.129@tcp:250:kworker.0 00000800:00000100:0.0:1787827662.210683:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787827662.210737:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827662/real 1787827662] req@ffff8cb998a8c780 x1874672572476288/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827687 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827662.210759:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8c780 x1874672572476288/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827687 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827662.210773:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827662.210778:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787827662.210783:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8c780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572476288:192.168.203.129@tcp:250:kworker.0 00000100:00100000:0.0:1787827662.210792:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8e580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572476416:192.168.203.129@tcp:38:kworker.0 00000800:00000100:0.0:1787827662.210816:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787827662.210830:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827662/real 1787827662] req@ffff8cb998a8e580 x1874672572476416/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827687 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827662.210843:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8e580 x1874672572476416/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827687 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827662.210873:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827662.210878:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787827662.210882:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8e580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572476416:192.168.203.129@tcp:38:kworker.0 00000100:00100000:0.0:1787827662.210933:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572476544:192.168.203.129@tcp:38:kworker.0 00000800:00000100:0.0:1787827662.210958:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787827662.210971:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827662/real 1787827662] req@ffff8cb998a8c000 x1874672572476544/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827687 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827662.210984:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8c000 x1874672572476544/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827687 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827662.210994:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827662.210998:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787827662.211002:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572476544:192.168.203.129@tcp:38:kworker.0 00010000:00010000:1.0:1787827663.042450:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787827664.066406:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787827665.090419:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787827666.114454:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787827667.138443:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787827667.330368:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827667.330392:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787827667.330398:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827667.330412:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 619 00000100:00080000:2.0:1787827667.330427:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827667.330437:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:2.0:1787827667.330488:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8f480 x1874672572476672/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787827667.330522:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827667.330526:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787827667.330529:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827667.330535:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 619 00000100:00080000:2.0:1787827667.330560:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827667.330571:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:1.0:1787827667.330572:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572476672:192.168.203.129@tcp:250:kworker.0 00000008:00100000:2.0:1787827667.330584:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787827667.330602:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8cb40 x1874672572476800/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787827667.330625:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827667.330629:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1787827667.330632:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827667.330636:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 619 00000100:00080000:2.0:1787827667.330643:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000800:00000100:1.0:1787827667.330651:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00080000:2.0:1787827667.330652:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:2.0:1787827667.330668:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787827667.330682:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8e940 x1874672572476928/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787827667.330704:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827667/real 1787827667] req@ffff8cb998a8f480 x1874672572476672/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827692 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787827667.330735:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8f480 x1874672572476672/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827692 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787827667.330747:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787827667.330752:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787827667.330756:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572476672:192.168.203.129@tcp:250:kworker.0 00000100:00100000:1.0:1787827667.330781:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8cb40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572476800:192.168.203.129@tcp:38:kworker.0 00000800:00000100:1.0:1787827667.330809:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787827667.330823:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827667/real 1787827667] req@ffff8cb998a8cb40 x1874672572476800/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827692 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787827667.330836:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8cb40 x1874672572476800/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827692 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787827667.330844:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787827667.330855:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787827667.330859:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8cb40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572476800:192.168.203.129@tcp:38:kworker.0 00000100:00100000:1.0:1787827667.330867:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8e940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572476928:192.168.203.129@tcp:38:kworker.0 00000800:00000100:1.0:1787827667.330904:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787827667.330917:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827667/real 1787827667] req@ffff8cb998a8e940 x1874672572476928/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827692 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787827667.330929:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8e940 x1874672572476928/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827692 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787827667.330938:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787827667.330943:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787827667.330947:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8e940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572476928:192.168.203.129@tcp:38:kworker.0 00010000:00010000:0.0:1787827668.162428:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787827669.186448:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787827670.210458:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787827671.234443:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787827672.258451:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787827672.450375:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787827672.450384:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787827672.450387:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787827672.450395:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 624 00000100:00080000:1.0:1787827672.450404:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787827672.450409:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:1.0:1787827672.450480:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f38b40 x1874672572477056/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787827672.450504:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787827672.450507:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787827672.450522:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787827672.450526:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 624 00000100:00080000:1.0:1787827672.450529:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787827672.450536:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:0.0:1787827672.450537:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f38b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572477056:192.168.203.129@tcp:250:kworker.0 00000008:00100000:1.0:1787827672.450545:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787827672.450558:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f39e00 x1874672572477184/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787827672.450567:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787827672.450569:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1787827672.450571:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787827672.450574:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 624 00000100:00080000:1.0:1787827672.450598:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787827672.450602:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:1.0:1787827672.450607:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000800:00000100:0.0:1787827672.450614:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787827672.450623:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f38f00 x1874672572477312/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787827672.450663:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827672/real 1787827672] req@ffff8cb999f38b40 x1874672572477056/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827697 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827672.450683:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f38b40 x1874672572477056/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827697 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827672.450701:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827672.450704:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787827672.450707:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f38b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572477056:192.168.203.129@tcp:250:kworker.0 00000100:00100000:0.0:1787827672.450727:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f39e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572477184:192.168.203.129@tcp:38:kworker.0 00000800:00000100:0.0:1787827672.450746:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787827672.450756:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827672/real 1787827672] req@ffff8cb999f39e00 x1874672572477184/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827697 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827672.450765:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f39e00 x1874672572477184/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827697 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827672.450770:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827672.450772:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787827672.450774:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f39e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572477184:192.168.203.129@tcp:38:kworker.0 00000100:00100000:0.0:1787827672.450778:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f38f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572477312:192.168.203.129@tcp:38:kworker.0 00000800:00000100:0.0:1787827672.450789:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787827672.450795:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827672/real 1787827672] req@ffff8cb999f38f00 x1874672572477312/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827697 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827672.450802:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f38f00 x1874672572477312/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827697 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827672.450807:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827672.450808:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787827672.450810:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f38f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572477312:192.168.203.129@tcp:38:kworker.0 00010000:00010000:0.0:1787827673.282399:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787827674.306410:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787827675.330372:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787827676.354515:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787827677.378409:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787827677.570339:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787827677.570347:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787827677.570351:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787827677.570359:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 629 00000100:00080000:1.0:1787827677.570368:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787827677.570391:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:1.0:1787827677.570432:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f39a40 x1874672572477440/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787827677.570473:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787827677.570476:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787827677.570478:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787827677.570481:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 629 00000100:00080000:1.0:1787827677.570501:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787827677.570506:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:1.0:1787827677.570515:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787827677.570524:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f39a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572477440:192.168.203.129@tcp:250:kworker.0 00000100:00100000:1.0:1787827677.570529:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f3b840 x1874672572477568/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787827677.570539:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787827677.570541:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1787827677.570543:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787827677.570546:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 629 00000100:00080000:1.0:1787827677.570564:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787827677.570566:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:1.0:1787827677.570570:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787827677.570579:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f38f00 x1874672572477696/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1787827677.570605:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787827677.570639:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827677/real 1787827677] req@ffff8cb999f39a40 x1874672572477440/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827702 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827677.570653:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f39a40 x1874672572477440/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827702 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827677.570673:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827677.570675:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787827677.570678:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f39a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572477440:192.168.203.129@tcp:250:kworker.0 00000100:00100000:0.0:1787827677.570699:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f3b840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572477568:192.168.203.129@tcp:38:kworker.0 00000800:00000100:0.0:1787827677.570715:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787827677.570723:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827677/real 1787827677] req@ffff8cb999f3b840 x1874672572477568/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827702 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827677.570732:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f3b840 x1874672572477568/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827702 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827677.570737:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827677.570755:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787827677.570759:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f3b840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572477568:192.168.203.129@tcp:38:kworker.0 00000100:00100000:0.0:1787827677.570790:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f38f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572477696:192.168.203.129@tcp:38:kworker.0 00000800:00000100:0.0:1787827677.570855:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787827677.570868:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827677/real 1787827677] req@ffff8cb999f38f00 x1874672572477696/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827702 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827677.570881:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f38f00 x1874672572477696/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827702 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827677.570890:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827677.570902:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787827677.570908:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f38f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572477696:192.168.203.129@tcp:38:kworker.0 00010000:00010000:1.0:1787827678.402407:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787827679.426448:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787827680.450542:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787827681.474383:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787827682.498589:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787827682.690430:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827682.690438:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787827682.690442:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827682.690451:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 634 00000100:00080000:2.0:1787827682.690461:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827682.690467:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:2.0:1787827682.690498:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8ed00 x1874672572477824/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787827682.690534:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827682.690552:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787827682.690553:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827682.690556:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 634 00000100:00080000:2.0:1787827682.690559:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827682.690562:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:2.0:1787827682.690571:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787827682.690581:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8f840 x1874672572477952/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787827682.690590:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827682.690592:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1787827682.690595:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1787827682.690597:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8ed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572477824:192.168.203.129@tcp:250:kworker.0 00000100:00080000:2.0:1787827682.690600:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 634 00000100:00080000:2.0:1787827682.690606:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827682.690610:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:2.0:1787827682.690615:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787827682.690629:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8e940 x1874672572478080/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1787827682.690656:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787827682.690691:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827682/real 1787827682] req@ffff8cb998a8ed00 x1874672572477824/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827707 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827682.690703:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8ed00 x1874672572477824/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827707 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827682.690711:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827682.690714:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787827682.690717:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8ed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572477824:192.168.203.129@tcp:250:kworker.0 00000100:00100000:0.0:1787827682.690736:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8f840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572477952:192.168.203.129@tcp:38:kworker.0 00000800:00000100:0.0:1787827682.690760:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787827682.690784:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827682/real 1787827682] req@ffff8cb998a8f840 x1874672572477952/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827707 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827682.690794:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8f840 x1874672572477952/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827707 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827682.690800:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827682.690802:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787827682.690805:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8f840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572477952:192.168.203.129@tcp:38:kworker.0 00000100:00100000:0.0:1787827682.690809:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8e940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572478080:192.168.203.129@tcp:38:kworker.0 00000800:00000100:0.0:1787827682.690824:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787827682.690837:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827682/real 1787827682] req@ffff8cb998a8e940 x1874672572478080/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827707 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827682.690845:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8e940 x1874672572478080/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827707 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827682.690850:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827682.690852:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787827682.690854:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8e940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572478080:192.168.203.129@tcp:38:kworker.0 00010000:00010000:0.0:1787827683.522549:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787827684.546423:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787827685.570357:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787827686.594402:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787827687.618374:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787827687.810374:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787827687.810387:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787827687.810414:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787827687.810427:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 639 00000100:00080000:0.0:1787827687.810455:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787827687.810461:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:0.0:1787827687.810492:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c69a40 x1874672572478208/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787827687.810545:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787827687.810549:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787827687.810551:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787827687.810555:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 639 00000100:00080000:0.0:1787827687.810558:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787827687.810562:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:0.0:1787827687.810572:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787827687.810584:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c6b840 x1874672572478336/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787827687.810595:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787827687.810600:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1787827687.810603:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1787827687.810606:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c69a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572478208:192.168.203.129@tcp:250:kworker.0 00000100:00080000:0.0:1787827687.810609:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 639 00000100:00080000:0.0:1787827687.810612:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787827687.810615:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:0.0:1787827687.810619:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787827687.810634:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c69e00 x1874672572478464/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1787827687.810725:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787827687.810832:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827687/real 1787827687] req@ffff8cb999c69a40 x1874672572478208/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827712 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787827687.810859:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c69a40 x1874672572478208/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827712 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787827687.810872:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787827687.810878:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787827687.810883:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c69a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572478208:192.168.203.129@tcp:250:kworker.0 00000100:00100000:1.0:1787827687.810911:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c6b840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572478336:192.168.203.129@tcp:38:kworker.0 00000800:00000100:1.0:1787827687.810944:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787827687.810964:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827687/real 1787827687] req@ffff8cb999c6b840 x1874672572478336/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827712 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787827687.810981:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c6b840 x1874672572478336/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827712 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787827687.810994:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787827687.811000:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787827687.811005:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c6b840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572478336:192.168.203.129@tcp:38:kworker.0 00000100:00100000:1.0:1787827687.811015:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c69e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572478464:192.168.203.129@tcp:38:kworker.0 00000800:00000100:1.0:1787827687.811046:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787827687.811078:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827687/real 1787827687] req@ffff8cb999c69e00 x1874672572478464/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827712 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787827687.811109:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c69e00 x1874672572478464/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827712 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787827687.811119:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787827687.811140:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787827687.811145:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c69e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572478464:192.168.203.129@tcp:38:kworker.0 00010000:00010000:0.0:1787827688.642386:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787827689.666386:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787827690.690392:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787827691.714451:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787827692.738518:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787827692.930440:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827692.930451:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787827692.930455:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827692.930465:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 645 00000100:00080000:2.0:1787827692.930475:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827692.930482:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:2.0:1787827692.930524:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8cb40 x1874672572478592/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787827692.930587:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827692.930608:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787827692.930610:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827692.930616:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 645 00000100:00080000:2.0:1787827692.930621:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827692.930626:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:2.0:1787827692.930638:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787827692.930657:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8f480 x1874672572478720/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787827692.930670:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827692.930674:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1787827692.930676:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827692.930718:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 645 00000100:00080000:2.0:1787827692.930724:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827692.930737:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:1.0:1787827692.930745:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8cb40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572478592:192.168.203.129@tcp:250:kworker.0 00000008:00100000:2.0:1787827692.930757:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787827692.930782:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8c000 x1874672572478848/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1787827692.930817:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787827692.930870:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827692/real 1787827692] req@ffff8cb998a8cb40 x1874672572478592/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827717 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787827692.930897:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8cb40 x1874672572478592/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827717 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787827692.930914:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787827692.930920:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787827692.930926:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8cb40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572478592:192.168.203.129@tcp:250:kworker.0 00000100:00100000:1.0:1787827692.930936:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572478720:192.168.203.129@tcp:38:kworker.0 00000800:00000100:1.0:1787827692.930966:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787827692.930987:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827692/real 1787827692] req@ffff8cb998a8f480 x1874672572478720/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827717 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787827692.931006:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8f480 x1874672572478720/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827717 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787827692.931072:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787827692.931077:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787827692.931082:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572478720:192.168.203.129@tcp:38:kworker.0 00000100:00100000:1.0:1787827692.931125:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572478848:192.168.203.129@tcp:38:kworker.0 00000800:00000100:1.0:1787827692.931184:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787827692.931203:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827692/real 1787827692] req@ffff8cb998a8c000 x1874672572478848/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827717 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787827692.931218:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8c000 x1874672572478848/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827717 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787827692.931226:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787827692.931229:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787827692.931251:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572478848:192.168.203.129@tcp:38:kworker.0 00010000:00010000:1.0:1787827693.762389:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787827694.786420:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787827695.810586:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787827696.834476:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787827697.858594:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787827698.050339:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827698.050347:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787827698.050350:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827698.050358:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 650 00000100:00080000:2.0:1787827698.050381:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827698.050393:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:2.0:1787827698.050426:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8e580 x1874672572478976/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787827698.050445:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827698.050448:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787827698.050449:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827698.050451:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 650 00000100:00080000:2.0:1787827698.050453:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827698.050456:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:2.0:1787827698.050464:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787827698.050473:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8c780 x1874672572479104/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787827698.050480:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1787827698.050483:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8e580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572478976:192.168.203.129@tcp:250:kworker.0 00000100:00080000:2.0:1787827698.050484:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1787827698.050488:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827698.050496:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 650 00000100:00080000:2.0:1787827698.050498:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827698.050500:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:2.0:1787827698.050503:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787827698.050511:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8d680 x1874672572479232/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1787827698.050552:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787827698.050596:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827698/real 1787827698] req@ffff8cb998a8e580 x1874672572478976/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827723 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787827698.050614:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8e580 x1874672572478976/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827723 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787827698.050622:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787827698.050630:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787827698.050633:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8e580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572478976:192.168.203.129@tcp:250:kworker.0 00000100:00100000:1.0:1787827698.050652:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8c780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572479104:192.168.203.129@tcp:38:kworker.0 00000800:00000100:1.0:1787827698.050692:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787827698.050705:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827698/real 1787827698] req@ffff8cb998a8c780 x1874672572479104/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827723 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787827698.050719:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8c780 x1874672572479104/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827723 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787827698.050728:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787827698.050731:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787827698.050734:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8c780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572479104:192.168.203.129@tcp:38:kworker.0 00000100:00100000:1.0:1787827698.050739:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8d680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572479232:192.168.203.129@tcp:38:kworker.0 00000800:00000100:1.0:1787827698.050754:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787827698.050765:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827698/real 1787827698] req@ffff8cb998a8d680 x1874672572479232/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827723 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787827698.050778:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8d680 x1874672572479232/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827723 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787827698.050787:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787827698.050790:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787827698.050793:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8d680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572479232:192.168.203.129@tcp:38:kworker.0 00010000:00010000:0.0:1787827698.882425:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787827699.906453:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787827700.930432:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787827701.954471:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787827702.978460:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787827703.170474:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827703.170499:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787827703.170504:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827703.170512:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 655 00000100:00080000:2.0:1787827703.170522:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827703.170527:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:2.0:1787827703.170557:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8cf00 x1874672572479360/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787827703.170579:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827703.170582:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787827703.170584:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827703.170587:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 655 00000100:00080000:2.0:1787827703.170590:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827703.170593:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:2.0:1787827703.170605:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787827703.170636:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8f0c0 x1874672572479488/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787827703.170664:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827703.170670:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00100000:1.0:1787827703.170672:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8cf00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572479360:192.168.203.129@tcp:250:kworker.0 00000100:00080000:2.0:1787827703.170673:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827703.170688:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 655 00000100:00080000:2.0:1787827703.170695:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827703.170699:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:2.0:1787827703.170706:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787827703.170738:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8e1c0 x1874672572479616/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1787827703.170795:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787827703.170866:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827703/real 1787827703] req@ffff8cb998a8cf00 x1874672572479360/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827728 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787827703.170890:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8cf00 x1874672572479360/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827728 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787827703.170916:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787827703.170922:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787827703.170928:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8cf00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572479360:192.168.203.129@tcp:250:kworker.0 00000100:00100000:1.0:1787827703.170966:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8f0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572479488:192.168.203.129@tcp:38:kworker.0 00000800:00000100:1.0:1787827703.171007:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787827703.171030:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827703/real 1787827703] req@ffff8cb998a8f0c0 x1874672572479488/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827728 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787827703.171046:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8f0c0 x1874672572479488/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827728 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787827703.171067:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787827703.171072:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787827703.171077:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8f0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572479488:192.168.203.129@tcp:38:kworker.0 00000100:00100000:1.0:1787827703.171087:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8e1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572479616:192.168.203.129@tcp:38:kworker.0 00000800:00000100:1.0:1787827703.171121:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787827703.171150:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827703/real 1787827703] req@ffff8cb998a8e1c0 x1874672572479616/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827728 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787827703.171182:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8e1c0 x1874672572479616/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827728 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787827703.171198:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787827703.171204:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787827703.171212:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8e1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572479616:192.168.203.129@tcp:38:kworker.0 00010000:00010000:0.0:1787827704.002480:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787827705.026497:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787827706.050425:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787827707.074479:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787827708.098373:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787827708.290341:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827708.290353:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787827708.290357:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827708.290364:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 660 00000100:00080000:2.0:1787827708.290374:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827708.290379:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:2.0:1787827708.290413:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8da40 x1874672572479744/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787827708.290438:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827708.290442:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787827708.290445:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827708.290449:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 660 00000100:00080000:2.0:1787827708.290454:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827708.290459:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:2.0:1787827708.290481:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787827708.290496:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8fc00 x1874672572479872/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787827708.290503:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8da40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572479744:192.168.203.129@tcp:250:kworker.0 00000100:00080000:2.0:1787827708.290517:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827708.290521:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1787827708.290523:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827708.290527:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 660 00000100:00080000:2.0:1787827708.290531:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827708.290534:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:2.0:1787827708.290552:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000800:00000100:1.0:1787827708.290563:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787827708.290567:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8de00 x1874672572480000/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787827708.290617:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827708/real 1787827708] req@ffff8cb998a8da40 x1874672572479744/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827733 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787827708.290632:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8da40 x1874672572479744/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827733 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787827708.290640:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787827708.290643:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787827708.290646:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8da40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572479744:192.168.203.129@tcp:250:kworker.0 00000100:00100000:1.0:1787827708.290664:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8fc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572479872:192.168.203.129@tcp:38:kworker.0 00000800:00000100:1.0:1787827708.290679:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787827708.290688:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827708/real 1787827708] req@ffff8cb998a8fc00 x1874672572479872/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827733 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787827708.290696:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8fc00 x1874672572479872/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827733 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787827708.290712:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787827708.290714:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787827708.290717:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8fc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572479872:192.168.203.129@tcp:38:kworker.0 00000100:00100000:1.0:1787827708.290721:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8de00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572480000:192.168.203.129@tcp:38:kworker.0 00000800:00000100:1.0:1787827708.290734:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787827708.290740:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827708/real 1787827708] req@ffff8cb998a8de00 x1874672572480000/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827733 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787827708.290747:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8de00 x1874672572480000/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827733 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787827708.290752:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787827708.290754:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787827708.290756:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8de00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572480000:192.168.203.129@tcp:38:kworker.0 00010000:00010000:1.0:1787827709.122472:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787827710.146571:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787827711.170477:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787827712.194433:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787827713.218484:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787827713.410338:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827713.410349:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787827713.410355:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827713.410367:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 665 00000100:00080000:2.0:1787827713.410380:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827713.410389:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:2.0:1787827713.410467:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8c3c0 x1874672572480128/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787827713.410512:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827713.410518:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787827713.410520:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827713.410525:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 665 00000100:00080000:2.0:1787827713.410530:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827713.410535:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:2.0:1787827713.410549:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787827713.410606:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8d2c0 x1874672572480256/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787827713.410614:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827713.410617:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1787827713.410618:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827713.410622:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 665 00000100:00100000:1.0:1787827713.410665:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8c3c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572480128:192.168.203.129@tcp:250:kworker.0 00000100:00080000:2.0:1787827713.410666:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827713.410682:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:2.0:1787827713.410689:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787827713.410706:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8de00 x1874672572480384/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787827713.410859:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8d2c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572480256:192.168.203.129@tcp:38:kworker.0 00000100:00100000:1.0:1787827713.410921:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8de00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572480384:192.168.203.129@tcp:38:kworker.0 00010000:00010000:0.0:1787827714.242429:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787827715.266398:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787827716.290494:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:2.0:1787827716.482476:0:2382:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.129:988 00000400:00000100:2.0:1787827716.482512:0:2382:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.129@tcp at host 192.168.203.129 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:2.0:1787827716.482525:0:2382:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.129@tcp 00000800:00000100:2.0:1787827716.482571:0:2382:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.129@tcp 00000800:00000100:2.0:1787827716.482583:0:2382:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.129@tcp 00000100:00100000:1.0:1787827716.482634:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827713/real 1787827716] req@ffff8cb998a8de00 x1874672572480384/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827738 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787827716.482683:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8de00 x1874672572480384/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827738 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787827716.482697:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787827716.482703:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787827716.482707:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8de00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572480384:192.168.203.129@tcp:38:kworker.0 00000100:00100000:1.0:1787827716.482715:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827713/real 1787827716] req@ffff8cb998a8c3c0 x1874672572480128/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827738 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787827716.482727:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8c3c0 x1874672572480128/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827738 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787827716.482734:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787827716.482737:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787827716.482740:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8c3c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572480128:192.168.203.129@tcp:250:kworker.0 00000100:00100000:1.0:1787827716.482762:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827713/real 1787827716] req@ffff8cb998a8d2c0 x1874672572480256/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827738 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787827716.482776:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8d2c0 x1874672572480256/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827738 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787827716.482785:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787827716.482788:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787827716.482791:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8d2c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572480256:192.168.203.129@tcp:38:kworker.0 00010000:00010000:0.0:1787827717.314542:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787827718.338858:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787827718.530395:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827718.530405:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787827718.530409:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827718.530420:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 670 00000100:00080000:2.0:1787827718.530431:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827718.530437:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:2.0:1787827718.530471:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8fc00 x1874672572480512/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787827718.530507:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827718.530514:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787827718.530529:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827718.530534:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 670 00000100:00080000:2.0:1787827718.530538:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827718.530543:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:1.0:1787827718.530582:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8fc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572480512:192.168.203.129@tcp:250:kworker.0 00000008:00100000:2.0:1787827718.530593:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787827718.530612:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8da40 x1874672572480640/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787827718.530625:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827718.530646:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1787827718.530658:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827718.530662:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 670 00000100:00080000:2.0:1787827718.530667:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827718.530672:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:2.0:1787827718.530677:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787827718.530690:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8e1c0 x1874672572480768/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1787827718.530693:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787827718.530751:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827718/real 1787827718] req@ffff8cb998a8fc00 x1874672572480512/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827743 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787827718.530771:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8fc00 x1874672572480512/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827743 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787827718.530789:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787827718.530794:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787827718.530798:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8fc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572480512:192.168.203.129@tcp:250:kworker.0 00000100:00100000:1.0:1787827718.530828:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8da40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572480640:192.168.203.129@tcp:38:kworker.0 00000800:00000100:1.0:1787827718.530854:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787827718.530870:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827718/real 1787827718] req@ffff8cb998a8da40 x1874672572480640/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827743 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787827718.530884:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8da40 x1874672572480640/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827743 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787827718.530893:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787827718.530896:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787827718.530901:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8da40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572480640:192.168.203.129@tcp:38:kworker.0 00000100:00100000:1.0:1787827718.530908:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8e1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572480768:192.168.203.129@tcp:38:kworker.0 00000800:00000100:1.0:1787827718.530931:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787827718.530945:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827718/real 1787827718] req@ffff8cb998a8e1c0 x1874672572480768/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827743 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787827718.530974:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8e1c0 x1874672572480768/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827743 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787827718.530983:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787827718.530987:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787827718.530991:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8e1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572480768:192.168.203.129@tcp:38:kworker.0 00010000:00010000:2.0:1787827719.363092:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787827720.386536:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787827721.410683:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787827722.434542:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787827723.458502:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787827723.650334:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827723.650345:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787827723.650349:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827723.650359:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 675 00000100:00080000:2.0:1787827723.650370:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827723.650387:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:2.0:1787827723.650417:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8f0c0 x1874672572480896/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787827723.650438:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827723.650448:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787827723.650450:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827723.650454:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 675 00000100:00080000:2.0:1787827723.650457:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827723.650461:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:2.0:1787827723.650469:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787827723.650471:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8f0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572480896:192.168.203.129@tcp:250:kworker.0 00000100:00100000:2.0:1787827723.650494:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8cf00 x1874672572481024/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787827723.650504:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827723.650507:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1787827723.650509:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827723.650512:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 675 00000100:00080000:2.0:1787827723.650516:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827723.650519:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:2.0:1787827723.650522:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787827723.650544:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8d680 x1874672572481152/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1787827723.650551:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787827723.650588:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827723/real 1787827723] req@ffff8cb998a8f0c0 x1874672572480896/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827748 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787827723.650602:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8f0c0 x1874672572480896/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827748 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787827723.650610:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787827723.650613:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787827723.650616:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8f0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572480896:192.168.203.129@tcp:250:kworker.0 00000100:00100000:1.0:1787827723.650634:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8cf00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572481024:192.168.203.129@tcp:38:kworker.0 00000800:00000100:1.0:1787827723.650649:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787827723.650657:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827723/real 1787827723] req@ffff8cb998a8cf00 x1874672572481024/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827748 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787827723.650665:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8cf00 x1874672572481024/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827748 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787827723.650681:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787827723.650691:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787827723.650697:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8cf00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572481024:192.168.203.129@tcp:38:kworker.0 00000100:00100000:1.0:1787827723.650704:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8d680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572481152:192.168.203.129@tcp:38:kworker.0 00000800:00000100:1.0:1787827723.650727:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787827723.650739:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827723/real 1787827723] req@ffff8cb998a8d680 x1874672572481152/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827748 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787827723.650751:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8d680 x1874672572481152/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827748 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787827723.650761:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787827723.650764:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787827723.650768:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8d680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572481152:192.168.203.129@tcp:38:kworker.0 00010000:00010000:1.0:1787827724.482377:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787827725.506420:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787827726.530537:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787827727.554504:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787827728.578490:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787827728.770467:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787827728.770481:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787827728.770487:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787827728.770501:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 680 00000100:00080000:0.0:1787827728.770516:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787827728.770525:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:0.0:1787827728.770590:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c68f00 x1874672572481280/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787827728.770654:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787827728.770658:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787827728.770660:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787827728.770664:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 680 00000100:00080000:0.0:1787827728.770668:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787827728.770672:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:0.0:1787827728.770685:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787827728.770697:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c6bc00 x1874672572481408/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787827728.770706:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787827728.770709:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1787827728.770711:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787827728.770714:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 680 00000100:00080000:0.0:1787827728.770716:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787827728.770719:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:0.0:1787827728.770722:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787827728.770731:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c6b480 x1874672572481536/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787827728.770796:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c68f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572481280:192.168.203.129@tcp:250:kworker.0 00000800:00000100:1.0:1787827728.770903:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787827728.770969:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827728/real 1787827728] req@ffff8cb999c68f00 x1874672572481280/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827753 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787827728.771002:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c68f00 x1874672572481280/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827753 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787827728.771021:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787827728.771048:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787827728.771056:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c68f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572481280:192.168.203.129@tcp:250:kworker.0 00000100:00100000:1.0:1787827728.771070:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c6bc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572481408:192.168.203.129@tcp:38:kworker.0 00000800:00000100:1.0:1787827728.771104:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787827728.771122:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827728/real 1787827728] req@ffff8cb999c6bc00 x1874672572481408/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827753 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787827728.771137:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c6bc00 x1874672572481408/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827753 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787827728.771146:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787827728.771150:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787827728.771154:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c6bc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572481408:192.168.203.129@tcp:38:kworker.0 00000100:00100000:1.0:1787827728.771161:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c6b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572481536:192.168.203.129@tcp:38:kworker.0 00000800:00000100:1.0:1787827728.771180:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787827728.771196:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827728/real 1787827728] req@ffff8cb999c6b480 x1874672572481536/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827753 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787827728.771217:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c6b480 x1874672572481536/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827753 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787827728.771232:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787827728.771237:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787827728.771243:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c6b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572481536:192.168.203.129@tcp:38:kworker.0 00010000:00010000:2.0:1787827729.602388:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787827730.626407:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787827731.650525:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787827732.674447:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787827733.698430:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787827733.890481:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787827733.890519:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787827733.890524:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787827733.890534:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 686 00000100:00080000:1.0:1787827733.890543:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787827733.890548:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:1.0:1787827733.890590:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f39e00 x1874672572481664/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787827733.890613:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787827733.890616:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787827733.890617:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787827733.890620:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 686 00000100:00080000:1.0:1787827733.890623:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787827733.890626:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:1.0:1787827733.890634:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787827733.890642:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f39e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572481664:192.168.203.129@tcp:250:kworker.0 00000100:00100000:1.0:1787827733.890644:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f38b40 x1874672572481792/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787827733.890664:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787827733.890667:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1787827733.890668:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787827733.890671:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 686 00000100:00080000:1.0:1787827733.890674:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787827733.890676:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:1.0:1787827733.890680:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787827733.890688:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f3b0c0 x1874672572481920/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1787827733.890708:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787827733.890741:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827733/real 1787827733] req@ffff8cb999f39e00 x1874672572481664/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827758 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827733.890765:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f39e00 x1874672572481664/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827758 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827733.890773:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827733.890781:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787827733.890784:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f39e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572481664:192.168.203.129@tcp:250:kworker.0 00000100:00100000:0.0:1787827733.890806:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f38b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572481792:192.168.203.129@tcp:38:kworker.0 00000800:00000100:0.0:1787827733.890822:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787827733.890831:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827733/real 1787827733] req@ffff8cb999f38b40 x1874672572481792/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827758 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827733.890840:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f38b40 x1874672572481792/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827758 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827733.890845:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827733.890847:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787827733.890850:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f38b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572481792:192.168.203.129@tcp:38:kworker.0 00000100:00100000:0.0:1787827733.890853:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f3b0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572481920:192.168.203.129@tcp:38:kworker.0 00000800:00000100:0.0:1787827733.890863:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787827733.890869:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827733/real 1787827733] req@ffff8cb999f3b0c0 x1874672572481920/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827758 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827733.890876:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f3b0c0 x1874672572481920/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827758 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827733.890890:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827733.890893:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787827733.890895:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f3b0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572481920:192.168.203.129@tcp:38:kworker.0 00010000:00010000:2.0:1787827734.722423:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787827735.746476:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787827736.770516:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787827737.794405:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787827738.818608:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787827739.010393:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827739.010404:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787827739.010409:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827739.010430:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 691 00000100:00080000:2.0:1787827739.010441:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827739.010447:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:2.0:1787827739.010482:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8c780 x1874672572482048/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787827739.010508:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827739.010511:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787827739.010517:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827739.010520:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 691 00000100:00080000:2.0:1787827739.010523:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827739.010526:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:2.0:1787827739.010535:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787827739.010544:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8e580 x1874672572482176/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787827739.010554:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1787827739.010556:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8c780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572482048:192.168.203.129@tcp:250:kworker.0 00000100:00080000:2.0:1787827739.010558:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1787827739.010562:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827739.010575:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 691 00000100:00080000:2.0:1787827739.010602:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827739.010607:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:2.0:1787827739.010612:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787827739.010623:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8c000 x1874672572482304/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1787827739.010632:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787827739.010677:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827739/real 1787827739] req@ffff8cb998a8c780 x1874672572482048/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827764 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827739.010697:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8c780 x1874672572482048/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827764 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827739.010705:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827739.010709:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787827739.010712:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8c780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572482048:192.168.203.129@tcp:250:kworker.0 00000100:00100000:0.0:1787827739.010732:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8e580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572482176:192.168.203.129@tcp:38:kworker.0 00000800:00000100:0.0:1787827739.010750:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787827739.010760:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827739/real 1787827739] req@ffff8cb998a8e580 x1874672572482176/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827764 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827739.010769:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8e580 x1874672572482176/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827764 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827739.010785:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827739.010787:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787827739.010797:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8e580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572482176:192.168.203.129@tcp:38:kworker.0 00000100:00100000:0.0:1787827739.010802:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572482304:192.168.203.129@tcp:38:kworker.0 00000800:00000100:0.0:1787827739.010819:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787827739.010827:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827739/real 1787827739] req@ffff8cb998a8c000 x1874672572482304/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827764 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827739.010836:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8c000 x1874672572482304/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827764 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827739.010842:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827739.010844:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787827739.010847:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572482304:192.168.203.129@tcp:38:kworker.0 00010000:00010000:2.0:1787827739.842431:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787827740.866554:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787827741.890385:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787827742.914435:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787827743.938418:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787827744.130403:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827744.130414:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787827744.130418:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827744.130430:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 696 00000100:00080000:2.0:1787827744.130441:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827744.130451:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:2.0:1787827744.130491:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8f480 x1874672572482432/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787827744.130525:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827744.130528:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787827744.130530:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827744.130536:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 696 00000100:00080000:2.0:1787827744.130541:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827744.130546:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:0.0:1787827744.130557:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572482432:192.168.203.129@tcp:250:kworker.0 00000008:00100000:2.0:1787827744.130559:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787827744.130580:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8cb40 x1874672572482560/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787827744.130612:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827744.130616:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1787827744.130619:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:0.0:1787827744.130625:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00080000:2.0:1787827744.130629:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 696 00000100:00080000:2.0:1787827744.130637:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827744.130643:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:2.0:1787827744.130650:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787827744.130666:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8e940 x1874672572482688/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787827744.130675:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827744/real 1787827744] req@ffff8cb998a8f480 x1874672572482432/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827769 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827744.130697:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8f480 x1874672572482432/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827769 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827744.130707:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827744.130711:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787827744.130715:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572482432:192.168.203.129@tcp:250:kworker.0 00000100:00100000:0.0:1787827744.130740:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8cb40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572482560:192.168.203.129@tcp:38:kworker.0 00000800:00000100:0.0:1787827744.130800:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787827744.130817:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827744/real 1787827744] req@ffff8cb998a8cb40 x1874672572482560/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827769 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827744.130832:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8cb40 x1874672572482560/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827769 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827744.130841:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827744.130845:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787827744.130848:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8cb40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572482560:192.168.203.129@tcp:38:kworker.0 00000100:00100000:0.0:1787827744.130856:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8e940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572482688:192.168.203.129@tcp:38:kworker.0 00000800:00000100:0.0:1787827744.130878:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787827744.130890:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827744/real 1787827744] req@ffff8cb998a8e940 x1874672572482688/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827769 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827744.130904:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8e940 x1874672572482688/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827769 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827744.130930:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827744.130934:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787827744.130938:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8e940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572482688:192.168.203.129@tcp:38:kworker.0 00010000:00010000:0.0:1787827744.962495:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787827745.986502:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787827747.010407:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787827748.034403:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787827749.058411:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787827749.250360:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827749.250373:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787827749.250378:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827749.250388:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 701 00000100:00080000:2.0:1787827749.250398:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827749.250407:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:2.0:1787827749.250466:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8f840 x1874672572482816/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787827749.250503:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827749.250518:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787827749.250521:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827749.250544:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 701 00000100:00080000:2.0:1787827749.250551:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827749.250569:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:0.0:1787827749.250579:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8f840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572482816:192.168.203.129@tcp:250:kworker.0 00000008:00100000:2.0:1787827749.250582:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787827749.250600:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8ed00 x1874672572482944/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787827749.250615:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827749.250619:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1787827749.250621:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827749.250627:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 701 00000100:00080000:2.0:1787827749.250631:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827749.250636:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:2.0:1787827749.250642:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000800:00000100:0.0:1787827749.250650:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787827749.250655:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8e940 x1874672572483072/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787827749.250709:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827749/real 1787827749] req@ffff8cb998a8f840 x1874672572482816/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827774 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827749.250743:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8f840 x1874672572482816/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827774 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827749.250753:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827749.250757:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787827749.250762:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8f840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572482816:192.168.203.129@tcp:250:kworker.0 00000100:00100000:0.0:1787827749.250793:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8ed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572482944:192.168.203.129@tcp:38:kworker.0 00000800:00000100:0.0:1787827749.250829:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787827749.250849:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827749/real 1787827749] req@ffff8cb998a8ed00 x1874672572482944/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827774 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827749.250861:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8ed00 x1874672572482944/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827774 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827749.250869:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827749.250872:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787827749.250876:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8ed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572482944:192.168.203.129@tcp:38:kworker.0 00000100:00100000:0.0:1787827749.250882:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8e940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572483072:192.168.203.129@tcp:38:kworker.0 00000800:00000100:0.0:1787827749.250903:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787827749.250912:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827749/real 1787827749] req@ffff8cb998a8e940 x1874672572483072/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827774 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827749.250923:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8e940 x1874672572483072/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827774 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827749.250944:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827749.250947:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787827749.250950:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8e940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572483072:192.168.203.129@tcp:38:kworker.0 00010000:00010000:2.0:1787827750.082610:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787827751.106427:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787827752.130384:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787827753.154425:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787827754.178420:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787827754.370392:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787827754.370403:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787827754.370407:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787827754.370416:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 706 00000100:00080000:1.0:1787827754.370425:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787827754.370431:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:1.0:1787827754.370462:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f392c0 x1874672572483200/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787827754.370484:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787827754.370487:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787827754.370488:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787827754.370491:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 706 00000100:00080000:1.0:1787827754.370494:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787827754.370496:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:1.0:1787827754.370503:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787827754.370511:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f3b480 x1874672572483328/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787827754.370520:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787827754.370522:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1787827754.370524:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787827754.370526:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 706 00000100:00080000:1.0:1787827754.370529:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787827754.370534:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:0.0:1787827754.370537:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f392c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572483200:192.168.203.129@tcp:250:kworker.0 00000008:00100000:1.0:1787827754.370557:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787827754.370571:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f3a580 x1874672572483456/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1787827754.370604:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787827754.370638:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827754/real 1787827754] req@ffff8cb999f392c0 x1874672572483200/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827779 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827754.370651:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f392c0 x1874672572483200/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827779 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827754.370660:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827754.370662:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787827754.370665:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f392c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572483200:192.168.203.129@tcp:250:kworker.0 00000100:00100000:0.0:1787827754.370671:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f3b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572483328:192.168.203.129@tcp:38:kworker.0 00000800:00000100:0.0:1787827754.370687:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787827754.370695:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827754/real 1787827754] req@ffff8cb999f3b480 x1874672572483328/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827779 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827754.370704:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f3b480 x1874672572483328/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827779 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827754.370721:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827754.370735:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787827754.370739:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f3b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572483328:192.168.203.129@tcp:38:kworker.0 00000100:00100000:0.0:1787827754.370766:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f3a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572483456:192.168.203.129@tcp:38:kworker.0 00000800:00000100:0.0:1787827754.370793:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787827754.370803:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827754/real 1787827754] req@ffff8cb999f3a580 x1874672572483456/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827779 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827754.370811:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f3a580 x1874672572483456/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827779 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827754.370817:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827754.370819:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787827754.370821:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f3a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572483456:192.168.203.129@tcp:38:kworker.0 00010000:00010000:2.0:1787827755.202395:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787827756.226388:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787827757.250415:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787827758.274399:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787827759.298371:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787827759.490329:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787827759.490340:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787827759.490344:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787827759.490353:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 711 00000100:00080000:1.0:1787827759.490365:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787827759.490372:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:1.0:1787827759.490409:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f3a1c0 x1874672572483584/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787827759.490438:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787827759.490443:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787827759.490445:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787827759.490449:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 711 00000100:00080000:1.0:1787827759.490454:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787827759.490459:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:1.0:1787827759.490469:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787827759.490478:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f3a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572483584:192.168.203.129@tcp:250:kworker.0 00000100:00100000:1.0:1787827759.490486:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f383c0 x1874672572483712/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787827759.490498:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787827759.490502:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1787827759.490505:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787827759.490510:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 711 00000100:00080000:1.0:1787827759.490527:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000800:00000100:0.0:1787827759.490533:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00080000:1.0:1787827759.490535:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:1.0:1787827759.490544:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787827759.490562:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f3a580 x1874672572483840/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787827759.490568:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827759/real 1787827759] req@ffff8cb999f3a1c0 x1874672572483584/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827784 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827759.490583:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f3a1c0 x1874672572483584/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827784 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827759.490592:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827759.490595:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787827759.490599:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f3a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572483584:192.168.203.129@tcp:250:kworker.0 00000100:00100000:0.0:1787827759.490620:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f383c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572483712:192.168.203.129@tcp:38:kworker.0 00000800:00000100:0.0:1787827759.490652:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787827759.490667:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827759/real 1787827759] req@ffff8cb999f383c0 x1874672572483712/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827784 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827759.490679:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f383c0 x1874672572483712/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827784 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827759.490688:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827759.490692:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787827759.490694:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f383c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572483712:192.168.203.129@tcp:38:kworker.0 00000100:00100000:0.0:1787827759.490699:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f3a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572483840:192.168.203.129@tcp:38:kworker.0 00000800:00000100:0.0:1787827759.490715:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787827759.490723:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827759/real 1787827759] req@ffff8cb999f3a580 x1874672572483840/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827784 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827759.490730:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f3a580 x1874672572483840/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827784 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827759.490736:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827759.490738:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787827759.490740:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f3a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572483840:192.168.203.129@tcp:38:kworker.0 00010000:00010000:0.0:1787827760.322431:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787827761.346479:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787827762.370404:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787827763.394417:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787827764.418385:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787827764.610395:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827764.610422:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787827764.610425:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827764.610434:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 716 00000100:00080000:2.0:1787827764.610443:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827764.610451:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:2.0:1787827764.610487:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8cb40 x1874672572483968/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787827764.610517:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827764.610520:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787827764.610522:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827764.610525:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 716 00000100:00080000:2.0:1787827764.610529:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827764.610533:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:2.0:1787827764.610542:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787827764.610556:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8f480 x1874672572484096/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787827764.610559:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8cb40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572483968:192.168.203.129@tcp:250:kworker.0 00000100:00080000:2.0:1787827764.610570:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827764.610574:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1787827764.610576:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827764.610579:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 716 00000100:00080000:2.0:1787827764.610582:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827764.610584:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:2.0:1787827764.610588:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787827764.610597:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8c000 x1874672572484224/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1787827764.610627:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787827764.610665:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827764/real 1787827764] req@ffff8cb998a8cb40 x1874672572483968/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827789 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827764.610688:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8cb40 x1874672572483968/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827789 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827764.610696:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827764.610700:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787827764.610703:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8cb40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572483968:192.168.203.129@tcp:250:kworker.0 00000100:00100000:0.0:1787827764.610727:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572484096:192.168.203.129@tcp:38:kworker.0 00000800:00000100:0.0:1787827764.610754:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787827764.610770:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827764/real 1787827764] req@ffff8cb998a8f480 x1874672572484096/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827789 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827764.610785:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8f480 x1874672572484096/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827789 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827764.610794:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827764.610796:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787827764.610799:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572484096:192.168.203.129@tcp:38:kworker.0 00000100:00100000:0.0:1787827764.610804:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572484224:192.168.203.129@tcp:38:kworker.0 00000800:00000100:0.0:1787827764.610822:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787827764.610831:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827764/real 1787827764] req@ffff8cb998a8c000 x1874672572484224/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827789 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827764.610844:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8c000 x1874672572484224/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827789 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827764.610868:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827764.610873:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787827764.610875:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572484224:192.168.203.129@tcp:38:kworker.0 00010000:00010000:2.0:1787827765.442421:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787827766.466409:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787827767.490393:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787827768.514396:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787827769.538484:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787827769.730338:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787827769.730351:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787827769.730355:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787827769.730366:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 721 00000100:00080000:1.0:1787827769.730379:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787827769.730386:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:1.0:1787827769.730424:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f3b480 x1874672572484352/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787827769.730449:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787827769.730452:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787827769.730453:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787827769.730456:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 721 00000100:00080000:1.0:1787827769.730459:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787827769.730462:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:1.0:1787827769.730472:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787827769.730484:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f392c0 x1874672572484480/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787827769.730486:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f3b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572484352:192.168.203.129@tcp:250:kworker.0 00000100:00080000:1.0:1787827769.730498:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787827769.730526:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1787827769.730528:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787827769.730532:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 721 00000100:00080000:1.0:1787827769.730535:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787827769.730539:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:1.0:1787827769.730543:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787827769.730553:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f3b0c0 x1874672572484608/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1787827769.730573:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787827769.730620:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827769/real 1787827769] req@ffff8cb999f3b480 x1874672572484352/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827794 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827769.730639:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f3b480 x1874672572484352/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827794 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827769.730661:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827769.730669:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787827769.730674:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f3b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572484352:192.168.203.129@tcp:250:kworker.0 00000100:00100000:0.0:1787827769.730703:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f392c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572484480:192.168.203.129@tcp:38:kworker.0 00000800:00000100:0.0:1787827769.730727:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787827769.730741:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827769/real 1787827769] req@ffff8cb999f392c0 x1874672572484480/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827794 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827769.730755:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f392c0 x1874672572484480/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827794 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827769.730764:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827769.730775:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787827769.730779:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f392c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572484480:192.168.203.129@tcp:38:kworker.0 00000100:00100000:0.0:1787827769.730786:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f3b0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572484608:192.168.203.129@tcp:38:kworker.0 00000800:00000100:0.0:1787827769.730807:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787827769.730818:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827769/real 1787827769] req@ffff8cb999f3b0c0 x1874672572484608/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827794 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827769.730831:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f3b0c0 x1874672572484608/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827794 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827769.730840:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827769.730844:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787827769.730847:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f3b0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572484608:192.168.203.129@tcp:38:kworker.0 00010000:00010000:2.0:1787827770.562463:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787827771.586477:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787827772.610464:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787827773.634435:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787827774.658443:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787827774.850369:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787827774.850378:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787827774.850399:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787827774.850409:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 727 00000100:00080000:1.0:1787827774.850418:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787827774.850426:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:1.0:1787827774.850460:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f38b40 x1874672572484736/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787827774.850487:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787827774.850491:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787827774.850493:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787827774.850497:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 727 00000100:00080000:1.0:1787827774.850500:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787827774.850504:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:1.0:1787827774.850514:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787827774.850524:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f39e00 x1874672572484864/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787827774.850529:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f38b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572484736:192.168.203.129@tcp:250:kworker.0 00000100:00080000:1.0:1787827774.850549:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787827774.850552:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1787827774.850554:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787827774.850558:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 727 00000100:00080000:1.0:1787827774.850562:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787827774.850566:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:1.0:1787827774.850571:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787827774.850582:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f38f00 x1874672572484992/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1787827774.850591:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787827774.850646:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827774/real 1787827774] req@ffff8cb999f38b40 x1874672572484736/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827799 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787827774.850659:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f38b40 x1874672572484736/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827799 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787827774.850683:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787827774.850686:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787827774.850689:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f38b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572484736:192.168.203.129@tcp:250:kworker.0 00000100:00100000:2.0:1787827774.850711:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f39e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572484864:192.168.203.129@tcp:38:kworker.0 00000800:00000100:2.0:1787827774.850731:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787827774.850741:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827774/real 1787827774] req@ffff8cb999f39e00 x1874672572484864/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827799 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787827774.850751:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f39e00 x1874672572484864/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827799 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787827774.850757:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787827774.850768:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787827774.850771:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f39e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572484864:192.168.203.129@tcp:38:kworker.0 00000100:00100000:2.0:1787827774.850778:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f38f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572484992:192.168.203.129@tcp:38:kworker.0 00000800:00000100:2.0:1787827774.850819:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787827774.850833:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827774/real 1787827774] req@ffff8cb999f38f00 x1874672572484992/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827799 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787827774.850848:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f38f00 x1874672572484992/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827799 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787827774.850858:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787827774.850862:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787827774.850867:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f38f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572484992:192.168.203.129@tcp:38:kworker.0 00010000:00010000:0.0:1787827775.682414:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787827776.706450:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787827777.730431:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787827778.754386:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787827779.778388:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787827779.970365:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827779.970390:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787827779.970395:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827779.970406:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 732 00000100:00080000:2.0:1787827779.970417:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827779.970425:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:2.0:1787827779.970480:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8e580 x1874672572485120/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787827779.970510:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827779.970514:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787827779.970530:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827779.970534:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 732 00000100:00080000:2.0:1787827779.970538:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827779.970542:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:2.0:1787827779.970551:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787827779.970562:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8e580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572485120:192.168.203.129@tcp:250:kworker.0 00000100:00100000:2.0:1787827779.970563:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8c780 x1874672572485248/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787827779.970578:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827779.970580:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1787827779.970581:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827779.970584:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 732 00000100:00080000:2.0:1787827779.970587:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827779.970589:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:2.0:1787827779.970592:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787827779.970601:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8d680 x1874672572485376/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787827779.970671:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8c780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572485248:192.168.203.129@tcp:38:kworker.0 00000100:00100000:0.0:1787827779.970705:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8d680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572485376:192.168.203.129@tcp:38:kworker.0 00010000:00010000:2.0:1787827780.802446:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787827781.826470:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787827782.850493:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:3.0:1787827783.042496:0:2383:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.129:988 00000400:00000100:3.0:1787827783.042549:0:2383:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.129@tcp at host 192.168.203.129 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:3.0:1787827783.042571:0:2383:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.129@tcp 00000800:00000100:3.0:1787827783.042611:0:2383:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.129@tcp 00000800:00000100:3.0:1787827783.042638:0:2383:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.129@tcp 00000100:00100000:0.0:1787827783.042660:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827779/real 1787827783] req@ffff8cb998a8c780 x1874672572485248/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827804 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827783.042711:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8c780 x1874672572485248/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827804 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827783.042735:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827783.042743:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787827783.042748:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8c780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572485248:192.168.203.129@tcp:38:kworker.0 00000100:00100000:0.0:1787827783.042757:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827779/real 1787827783] req@ffff8cb998a8d680 x1874672572485376/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827804 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827783.042772:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8d680 x1874672572485376/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827804 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827783.042780:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827783.042784:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787827783.042788:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8d680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572485376:192.168.203.129@tcp:38:kworker.0 00000100:00100000:0.0:1787827783.042795:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827779/real 1787827783] req@ffff8cb998a8e580 x1874672572485120/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827804 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827783.042808:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8e580 x1874672572485120/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827804 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827783.042827:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827783.042831:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787827783.042834:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8e580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572485120:192.168.203.129@tcp:250:kworker.0 00010000:00010000:1.0:1787827783.874521:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787827784.899135:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787827785.090412:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827785.090427:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787827785.090434:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827785.090459:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 737 00000100:00080000:2.0:1787827785.090472:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827785.090479:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:2.0:1787827785.090539:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8cf00 x1874672572485504/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787827785.090572:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827785.090575:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787827785.090576:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827785.090579:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 737 00000100:00080000:2.0:1787827785.090582:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827785.090585:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:2.0:1787827785.090595:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787827785.090607:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8f0c0 x1874672572485632/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787827785.090615:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827785.090620:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1787827785.090622:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827785.090629:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 737 00000100:00100000:0.0:1787827785.090631:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8cf00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572485504:192.168.203.129@tcp:250:kworker.0 00000100:00080000:2.0:1787827785.090634:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827785.090642:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:2.0:1787827785.090647:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787827785.090662:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8e1c0 x1874672572485760/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1787827785.090719:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787827785.090786:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827785/real 1787827785] req@ffff8cb998a8cf00 x1874672572485504/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827810 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827785.090832:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8cf00 x1874672572485504/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827810 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827785.090845:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827785.090851:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787827785.090864:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8cf00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572485504:192.168.203.129@tcp:250:kworker.0 00000100:00100000:0.0:1787827785.090874:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8f0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572485632:192.168.203.129@tcp:38:kworker.0 00000800:00000100:0.0:1787827785.090904:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787827785.090927:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827785/real 1787827785] req@ffff8cb998a8f0c0 x1874672572485632/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827810 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827785.090942:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8f0c0 x1874672572485632/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827810 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827785.090951:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827785.090965:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787827785.090970:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8f0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572485632:192.168.203.129@tcp:38:kworker.0 00000100:00100000:0.0:1787827785.091017:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8e1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572485760:192.168.203.129@tcp:38:kworker.0 00000800:00000100:0.0:1787827785.091043:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787827785.091073:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827785/real 1787827785] req@ffff8cb998a8e1c0 x1874672572485760/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827810 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827785.091084:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8e1c0 x1874672572485760/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827810 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827785.091101:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827785.091105:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787827785.091109:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8e1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572485760:192.168.203.129@tcp:38:kworker.0 00010000:00010000:1.0:1787827785.922780:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787827786.946672:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787827787.970665:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787827788.994451:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787827790.018797:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787827790.210358:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827790.210369:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787827790.210373:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827790.210382:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 742 00000100:00080000:3.0:1787827790.210390:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827790.210396:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:3.0:1787827790.210427:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb992316580 x1874672572485888/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787827790.210452:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827790.210455:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787827790.210456:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827790.210459:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 742 00000100:00080000:3.0:1787827790.210461:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827790.210463:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:3.0:1787827790.210471:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787827790.210484:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb992314780 x1874672572486016/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787827790.210497:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1787827790.210499:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb992316580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572485888:192.168.203.129@tcp:250:kworker.0 00000100:00080000:3.0:1787827790.210506:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1787827790.210510:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827790.210516:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 742 00000100:00080000:3.0:1787827790.210521:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827790.210526:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:3.0:1787827790.210546:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787827790.210558:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb992314b40 x1874672572486144/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1787827790.210567:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787827790.210611:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827790/real 1787827790] req@ffff8cb992316580 x1874672572485888/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827815 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827790.210625:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb992316580 x1874672572485888/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827815 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827790.210633:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827790.210635:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787827790.210638:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb992316580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572485888:192.168.203.129@tcp:250:kworker.0 00000100:00100000:0.0:1787827790.210657:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb992314780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572486016:192.168.203.129@tcp:38:kworker.0 00000800:00000100:0.0:1787827790.210674:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787827790.210682:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827790/real 1787827790] req@ffff8cb992314780 x1874672572486016/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827815 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827790.210690:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb992314780 x1874672572486016/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827815 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827790.210696:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827790.210698:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787827790.210700:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb992314780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572486016:192.168.203.129@tcp:38:kworker.0 00000100:00100000:0.0:1787827790.210710:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb992314b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572486144:192.168.203.129@tcp:38:kworker.0 00000800:00000100:0.0:1787827790.210724:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787827790.210731:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827790/real 1787827790] req@ffff8cb992314b40 x1874672572486144/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827815 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827790.210739:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb992314b40 x1874672572486144/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827815 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827790.210744:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827790.210745:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787827790.210747:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb992314b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572486144:192.168.203.129@tcp:38:kworker.0 00010000:00010000:2.0:1787827791.043082:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787827792.066510:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787827793.090375:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787827794.114433:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787827795.138471:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787827795.330334:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787827795.330343:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787827795.330347:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787827795.330356:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 747 00000100:00080000:0.0:1787827795.330365:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787827795.330371:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:0.0:1787827795.330404:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c69680 x1874672572486272/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787827795.330431:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787827795.330435:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787827795.330438:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787827795.330442:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 747 00000100:00080000:0.0:1787827795.330460:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787827795.330490:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:0.0:1787827795.330510:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787827795.330512:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c69680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572486272:192.168.203.129@tcp:250:kworker.0 00000100:00100000:0.0:1787827795.330542:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c68000 x1874672572486400/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787827795.330556:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787827795.330560:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1787827795.330562:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787827795.330565:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 747 00000100:00080000:0.0:1787827795.330570:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787827795.330573:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:0.0:1787827795.330577:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000800:00000100:2.0:1787827795.330598:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787827795.330604:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c6a1c0 x1874672572486528/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787827795.330675:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827795/real 1787827795] req@ffff8cb999c69680 x1874672572486272/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827820 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787827795.330715:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c69680 x1874672572486272/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827820 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787827795.330728:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787827795.330733:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787827795.330738:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c69680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572486272:192.168.203.129@tcp:250:kworker.0 00000100:00100000:2.0:1787827795.330797:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c68000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572486400:192.168.203.129@tcp:38:kworker.0 00000800:00000100:2.0:1787827795.330829:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787827795.330845:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827795/real 1787827795] req@ffff8cb999c68000 x1874672572486400/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827820 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787827795.330872:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c68000 x1874672572486400/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827820 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787827795.330879:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787827795.330882:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787827795.330885:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c68000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572486400:192.168.203.129@tcp:38:kworker.0 00000100:00100000:2.0:1787827795.330890:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c6a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572486528:192.168.203.129@tcp:38:kworker.0 00000800:00000100:2.0:1787827795.330909:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787827795.330918:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827795/real 1787827795] req@ffff8cb999c6a1c0 x1874672572486528/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827820 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787827795.330961:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c6a1c0 x1874672572486528/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827820 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787827795.330972:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787827795.330976:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787827795.330980:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c6a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572486528:192.168.203.129@tcp:38:kworker.0 00010000:00010000:1.0:1787827796.162488:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787827797.186508:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787827798.210458:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787827799.234460:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787827800.258411:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787827800.450411:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827800.450423:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787827800.450428:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827800.450441:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 752 00000100:00080000:2.0:1787827800.450455:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827800.450464:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:2.0:1787827800.450514:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8da40 x1874672572486656/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787827800.450548:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827800.450551:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787827800.450553:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827800.450556:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 752 00000100:00080000:2.0:1787827800.450559:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827800.450562:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:2.0:1787827800.450574:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787827800.450591:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8da40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572486656:192.168.203.129@tcp:250:kworker.0 00000100:00100000:2.0:1787827800.450612:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8fc00 x1874672572486784/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787827800.450622:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827800.450625:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1787827800.450627:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827800.450630:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 752 00000100:00080000:2.0:1787827800.450634:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827800.450637:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:2.0:1787827800.450642:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787827800.450651:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8d2c0 x1874672572486912/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1787827800.450679:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787827800.450729:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827800/real 1787827800] req@ffff8cb998a8da40 x1874672572486656/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827825 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827800.450746:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8da40 x1874672572486656/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827825 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827800.450756:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827800.450760:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787827800.450772:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8da40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572486656:192.168.203.129@tcp:250:kworker.0 00000100:00100000:0.0:1787827800.450800:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8fc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572486784:192.168.203.129@tcp:38:kworker.0 00000800:00000100:0.0:1787827800.450821:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787827800.450830:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827800/real 1787827800] req@ffff8cb998a8fc00 x1874672572486784/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827825 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827800.450840:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8fc00 x1874672572486784/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827825 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827800.450847:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827800.450850:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787827800.450852:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8fc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572486784:192.168.203.129@tcp:38:kworker.0 00000100:00100000:0.0:1787827800.450857:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8d2c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572486912:192.168.203.129@tcp:38:kworker.0 00000800:00000100:0.0:1787827800.450872:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787827800.450878:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827800/real 1787827800] req@ffff8cb998a8d2c0 x1874672572486912/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827825 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827800.450887:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8d2c0 x1874672572486912/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827825 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827800.450893:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827800.450895:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787827800.450897:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8d2c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572486912:192.168.203.129@tcp:38:kworker.0 00010000:00010000:0.0:1787827801.282498:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787827802.306465:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787827803.330413:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787827804.354548:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787827805.378508:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787827805.570358:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827805.570390:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787827805.570397:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827805.570412:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 757 00000100:00080000:2.0:1787827805.570426:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827805.570438:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:2.0:1787827805.570492:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8c3c0 x1874672572487040/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787827805.570543:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827805.570551:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787827805.570555:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827805.570563:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 757 00000100:00080000:2.0:1787827805.570591:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827805.570600:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:2.0:1787827805.570617:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787827805.570641:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8de00 x1874672572487168/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787827805.570672:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8c3c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572487040:192.168.203.129@tcp:250:kworker.0 00000100:00080000:2.0:1787827805.570677:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827805.570685:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1787827805.570688:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827805.570694:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 757 00000100:00080000:2.0:1787827805.570701:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827805.570709:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:2.0:1787827805.570721:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787827805.570746:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8d2c0 x1874672572487296/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1787827805.570761:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787827805.570850:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827805/real 1787827805] req@ffff8cb998a8c3c0 x1874672572487040/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827830 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827805.570876:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8c3c0 x1874672572487040/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827830 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827805.570891:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827805.570896:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787827805.570901:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8c3c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572487040:192.168.203.129@tcp:250:kworker.0 00000100:00100000:0.0:1787827805.570982:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8de00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572487168:192.168.203.129@tcp:38:kworker.0 00000800:00000100:0.0:1787827805.571031:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787827805.571047:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827805/real 1787827805] req@ffff8cb998a8de00 x1874672572487168/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827830 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827805.571063:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8de00 x1874672572487168/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827830 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827805.571074:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827805.571078:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787827805.571082:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8de00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572487168:192.168.203.129@tcp:38:kworker.0 00000100:00100000:0.0:1787827805.571090:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8d2c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572487296:192.168.203.129@tcp:38:kworker.0 00000800:00000100:0.0:1787827805.571109:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787827805.571119:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827805/real 1787827805] req@ffff8cb998a8d2c0 x1874672572487296/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827830 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827805.571175:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8d2c0 x1874672572487296/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827830 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827805.571185:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827805.571188:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787827805.571192:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8d2c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572487296:192.168.203.129@tcp:38:kworker.0 00010000:00010000:2.0:1787827806.402568:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787827807.426510:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787827808.450499:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787827809.474412:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787827810.498380:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787827810.690378:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787827810.690386:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787827810.690390:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787827810.690400:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 762 00000100:00080000:0.0:1787827810.690412:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787827810.690420:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:0.0:1787827810.690457:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c6a580 x1874672572487424/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787827810.690485:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787827810.690488:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787827810.690489:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787827810.690492:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 762 00000100:00080000:0.0:1787827810.690495:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787827810.690499:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:0.0:1787827810.690509:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787827810.690518:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c6a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572487424:192.168.203.129@tcp:250:kworker.0 00000100:00100000:0.0:1787827810.690545:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c692c0 x1874672572487552/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787827810.690557:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787827810.690561:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1787827810.690564:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787827810.690568:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 762 00000100:00080000:0.0:1787827810.690573:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000800:00000100:2.0:1787827810.690575:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00080000:0.0:1787827810.690581:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:0.0:1787827810.690588:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787827810.690603:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c6a1c0 x1874672572487680/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787827810.690614:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827810/real 1787827810] req@ffff8cb999c6a580 x1874672572487424/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827835 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787827810.690629:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c6a580 x1874672572487424/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827835 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787827810.690638:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787827810.690641:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787827810.690652:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c6a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572487424:192.168.203.129@tcp:250:kworker.0 00000100:00100000:2.0:1787827810.690675:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c692c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572487552:192.168.203.129@tcp:38:kworker.0 00000800:00000100:2.0:1787827810.690699:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787827810.690709:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827810/real 1787827810] req@ffff8cb999c692c0 x1874672572487552/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827835 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787827810.690720:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c692c0 x1874672572487552/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827835 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787827810.690730:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787827810.690733:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787827810.690737:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c692c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572487552:192.168.203.129@tcp:38:kworker.0 00000100:00100000:2.0:1787827810.690745:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c6a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572487680:192.168.203.129@tcp:38:kworker.0 00000800:00000100:2.0:1787827810.690775:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787827810.690790:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827810/real 1787827810] req@ffff8cb999c6a1c0 x1874672572487680/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827835 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787827810.690800:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c6a1c0 x1874672572487680/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827835 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787827810.690807:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787827810.690811:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787827810.690813:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c6a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572487680:192.168.203.129@tcp:38:kworker.0 00010000:00010000:1.0:1787827811.522424:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787827812.546435:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787827813.570506:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787827814.594494:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787827815.618493:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787827815.810470:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827815.810481:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787827815.810487:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827815.810499:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 767 00000100:00080000:3.0:1787827815.810513:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827815.810523:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:3.0:1787827815.810579:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb992317480 x1874672572487808/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787827815.810625:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827815.810632:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787827815.810636:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827815.810643:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 767 00000100:00080000:3.0:1787827815.810651:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827815.810660:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:3.0:1787827815.810677:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787827815.810691:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb992317480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572487808:192.168.203.129@tcp:250:kworker.0 00000100:00100000:3.0:1787827815.810706:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb992314f00 x1874672572487936/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787827815.810724:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827815.810728:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1787827815.810732:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827815.810737:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 767 00000100:00080000:3.0:1787827815.810766:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827815.810772:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000800:00000100:2.0:1787827815.810781:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000008:00100000:3.0:1787827815.810782:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787827815.810803:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb992315e00 x1874672572488064/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787827815.810896:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827815/real 1787827815] req@ffff8cb992317480 x1874672572487808/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827840 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787827815.810933:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb992317480 x1874672572487808/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827840 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787827815.810953:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787827815.810960:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787827815.810967:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb992317480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572487808:192.168.203.129@tcp:250:kworker.0 00000100:00100000:2.0:1787827815.811007:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb992314f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572487936:192.168.203.129@tcp:38:kworker.0 00000800:00000100:2.0:1787827815.811054:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787827815.811080:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827815/real 1787827815] req@ffff8cb992314f00 x1874672572487936/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827840 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787827815.811097:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb992314f00 x1874672572487936/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827840 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787827815.811108:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787827815.811112:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787827815.811116:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb992314f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572487936:192.168.203.129@tcp:38:kworker.0 00000100:00100000:2.0:1787827815.811124:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb992315e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572488064:192.168.203.129@tcp:38:kworker.0 00000800:00000100:2.0:1787827815.811150:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787827815.811163:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827815/real 1787827815] req@ffff8cb992315e00 x1874672572488064/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827840 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787827815.811176:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb992315e00 x1874672572488064/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827840 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787827815.811204:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787827815.811209:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787827815.811213:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb992315e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572488064:192.168.203.129@tcp:38:kworker.0 00010000:00010000:0.0:1787827816.642532:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787827817.666415:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787827818.690464:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787827819.714397:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787827820.738548:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787827820.930394:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827820.930402:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787827820.930406:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827820.930415:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 773 00000100:00080000:2.0:1787827820.930423:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827820.930429:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:2.0:1787827820.930463:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8fc00 x1874672572488192/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787827820.930487:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827820.930490:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787827820.930492:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827820.930494:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 773 00000100:00080000:2.0:1787827820.930497:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827820.930499:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:2.0:1787827820.930507:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787827820.930516:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8da40 x1874672572488320/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787827820.930525:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827820.930530:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00100000:0.0:1787827820.930531:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8fc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572488192:192.168.203.129@tcp:250:kworker.0 00000100:00080000:2.0:1787827820.930544:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827820.930549:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 773 00000100:00080000:2.0:1787827820.930552:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827820.930555:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:2.0:1787827820.930560:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787827820.930570:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8e1c0 x1874672572488448/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1787827820.930590:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787827820.930634:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827820/real 1787827820] req@ffff8cb998a8fc00 x1874672572488192/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827845 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827820.930648:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8fc00 x1874672572488192/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827845 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827820.930659:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827820.930671:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787827820.930676:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8fc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572488192:192.168.203.129@tcp:250:kworker.0 00000100:00100000:0.0:1787827820.930700:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8da40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572488320:192.168.203.129@tcp:38:kworker.0 00000800:00000100:0.0:1787827820.930721:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787827820.930731:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827820/real 1787827820] req@ffff8cb998a8da40 x1874672572488320/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827845 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827820.930740:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8da40 x1874672572488320/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827845 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827820.930745:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827820.930748:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787827820.930750:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8da40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572488320:192.168.203.129@tcp:38:kworker.0 00000100:00100000:0.0:1787827820.930754:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8e1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572488448:192.168.203.129@tcp:38:kworker.0 00000800:00000100:0.0:1787827820.930767:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787827820.930776:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827820/real 1787827820] req@ffff8cb998a8e1c0 x1874672572488448/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827845 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827820.930786:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8e1c0 x1874672572488448/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827845 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827820.930792:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827820.930794:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787827820.930796:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8e1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572488448:192.168.203.129@tcp:38:kworker.0 00010000:00010000:0.0:1787827821.762431:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787827822.786406:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787827823.810498:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787827824.834488:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787827825.858488:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787827826.050375:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827826.050388:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787827826.050394:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827826.050406:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 778 00000100:00080000:2.0:1787827826.050420:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827826.050439:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:2.0:1787827826.050503:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8f0c0 x1874672572488576/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787827826.050542:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827826.050547:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787827826.050549:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827826.050554:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 778 00000100:00080000:2.0:1787827826.050559:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827826.050563:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:2.0:1787827826.050576:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787827826.050590:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8cf00 x1874672572488704/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787827826.050642:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827826.050647:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1787827826.050650:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827826.050656:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 778 00000100:00080000:2.0:1787827826.050661:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827826.050667:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:2.0:1787827826.050674:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787827826.050686:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8f0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572488576:192.168.203.129@tcp:250:kworker.0 00000100:00100000:2.0:1787827826.050693:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8e580 x1874672572488832/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1787827826.050825:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787827826.050882:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827826/real 1787827826] req@ffff8cb998a8f0c0 x1874672572488576/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827851 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827826.050905:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8f0c0 x1874672572488576/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827851 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827826.050920:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827826.050925:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787827826.050931:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8f0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572488576:192.168.203.129@tcp:250:kworker.0 00000100:00100000:0.0:1787827826.050940:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8cf00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572488704:192.168.203.129@tcp:38:kworker.0 00000800:00000100:0.0:1787827826.050967:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787827826.050981:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827826/real 1787827826] req@ffff8cb998a8cf00 x1874672572488704/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827851 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827826.050996:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8cf00 x1874672572488704/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827851 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827826.051006:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827826.051009:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787827826.051013:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8cf00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572488704:192.168.203.129@tcp:38:kworker.0 00000100:00100000:0.0:1787827826.051048:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8e580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572488832:192.168.203.129@tcp:38:kworker.0 00000800:00000100:0.0:1787827826.051110:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787827826.051134:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827826/real 1787827826] req@ffff8cb998a8e580 x1874672572488832/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827851 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827826.051159:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8e580 x1874672572488832/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827851 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827826.051200:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827826.051204:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787827826.051208:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8e580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572488832:192.168.203.129@tcp:38:kworker.0 00010000:00010000:1.0:1787827826.882409:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787827827.906411:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787827828.930420:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787827829.954453:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787827830.978456:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787827831.170376:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787827831.170389:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787827831.170394:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787827831.170405:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 783 00000100:00080000:0.0:1787827831.170419:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787827831.170427:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:0.0:1787827831.170476:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c68000 x1874672572488960/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787827831.170513:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787827831.170517:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787827831.170518:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787827831.170521:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 783 00000100:00080000:0.0:1787827831.170526:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787827831.170553:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:2.0:1787827831.170563:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c68000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572488960:192.168.203.129@tcp:250:kworker.0 00000008:00100000:0.0:1787827831.170568:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787827831.170584:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c69680 x1874672572489088/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787827831.170598:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787827831.170601:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1787827831.170603:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787827831.170607:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 783 00000100:00080000:0.0:1787827831.170612:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787827831.170616:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:0.0:1787827831.170620:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787827831.170630:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c6b480 x1874672572489216/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1787827831.170651:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787827831.170695:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827831/real 1787827831] req@ffff8cb999c68000 x1874672572488960/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827856 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787827831.170709:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c68000 x1874672572488960/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827856 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787827831.170719:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787827831.170722:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787827831.170725:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c68000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572488960:192.168.203.129@tcp:250:kworker.0 00000100:00100000:2.0:1787827831.170762:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c69680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572489088:192.168.203.129@tcp:38:kworker.0 00000800:00000100:2.0:1787827831.170790:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787827831.170800:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827831/real 1787827831] req@ffff8cb999c69680 x1874672572489088/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827856 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787827831.170821:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c69680 x1874672572489088/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827856 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787827831.170827:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787827831.170836:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787827831.170839:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c69680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572489088:192.168.203.129@tcp:38:kworker.0 00000100:00100000:2.0:1787827831.170844:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c6b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572489216:192.168.203.129@tcp:38:kworker.0 00000800:00000100:2.0:1787827831.170860:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787827831.170868:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827831/real 1787827831] req@ffff8cb999c6b480 x1874672572489216/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827856 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787827831.170876:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c6b480 x1874672572489216/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827856 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787827831.170882:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787827831.170884:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787827831.170886:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c6b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572489216:192.168.203.129@tcp:38:kworker.0 00010000:00010000:0.0:1787827832.002436:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787827833.026468:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787827834.050388:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787827835.074428:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787827836.098430:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787827836.290315:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827836.290332:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787827836.290336:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827836.290344:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 788 00000100:00080000:3.0:1787827836.290352:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827836.290358:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:3.0:1787827836.290386:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb992316940 x1874672572489344/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787827836.290407:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827836.290410:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787827836.290412:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827836.290414:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 788 00000100:00080000:3.0:1787827836.290416:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827836.290418:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:3.0:1787827836.290425:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787827836.290433:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb9923170c0 x1874672572489472/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787827836.290440:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827836.290456:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1787827836.290458:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1787827836.290460:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb992316940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572489344:192.168.203.129@tcp:250:kworker.0 00000100:00080000:3.0:1787827836.290463:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 788 00000100:00080000:3.0:1787827836.290466:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827836.290470:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:3.0:1787827836.290477:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787827836.290487:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb9923161c0 x1874672572489600/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1787827836.290539:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787827836.290580:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827836/real 1787827836] req@ffff8cb992316940 x1874672572489344/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827861 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787827836.290599:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb992316940 x1874672572489344/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827861 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787827836.290631:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787827836.290635:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787827836.290640:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb992316940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572489344:192.168.203.129@tcp:250:kworker.0 00000100:00100000:2.0:1787827836.290648:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb9923170c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572489472:192.168.203.129@tcp:38:kworker.0 00000800:00000100:2.0:1787827836.290676:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787827836.290692:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827836/real 1787827836] req@ffff8cb9923170c0 x1874672572489472/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827861 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787827836.290723:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb9923170c0 x1874672572489472/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827861 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787827836.290731:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787827836.290735:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787827836.290739:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb9923170c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572489472:192.168.203.129@tcp:38:kworker.0 00000100:00100000:2.0:1787827836.290766:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb9923161c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572489600:192.168.203.129@tcp:38:kworker.0 00000800:00000100:2.0:1787827836.290790:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787827836.290804:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827836/real 1787827836] req@ffff8cb9923161c0 x1874672572489600/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827861 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787827836.290818:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb9923161c0 x1874672572489600/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827861 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787827836.290827:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787827836.290830:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787827836.290834:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb9923161c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572489600:192.168.203.129@tcp:38:kworker.0 00010000:00010000:0.0:1787827837.122458:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787827838.146397:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787827839.170438:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787827840.194473:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787827841.218443:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787827841.410455:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787827841.410466:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787827841.410470:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787827841.410481:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 793 00000100:00080000:1.0:1787827841.410494:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787827841.410500:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:1.0:1787827841.410553:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f3b840 x1874672572489728/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787827841.410581:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787827841.410584:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787827841.410585:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787827841.410588:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 793 00000100:00080000:1.0:1787827841.410590:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787827841.410593:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:1.0:1787827841.410602:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787827841.410613:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f39a40 x1874672572489856/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787827841.410624:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787827841.410627:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1787827841.410629:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787827841.410633:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 793 00000100:00080000:1.0:1787827841.410637:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787827841.410642:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:2.0:1787827841.410685:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f3b840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572489728:192.168.203.129@tcp:250:kworker.0 00000008:00100000:1.0:1787827841.410686:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787827841.410707:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f38f00 x1874672572489984/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1787827841.410765:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787827841.410817:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827841/real 1787827841] req@ffff8cb999f3b840 x1874672572489728/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827866 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787827841.410833:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f3b840 x1874672572489728/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827866 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787827841.410843:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787827841.410847:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787827841.410851:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f3b840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572489728:192.168.203.129@tcp:250:kworker.0 00000100:00100000:2.0:1787827841.410857:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f39a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572489856:192.168.203.129@tcp:38:kworker.0 00000800:00000100:2.0:1787827841.410874:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787827841.410882:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827841/real 1787827841] req@ffff8cb999f39a40 x1874672572489856/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827866 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787827841.410890:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f39a40 x1874672572489856/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827866 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787827841.410896:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787827841.410898:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787827841.410901:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f39a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572489856:192.168.203.129@tcp:38:kworker.0 00000100:00100000:2.0:1787827841.410924:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f38f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572489984:192.168.203.129@tcp:38:kworker.0 00000800:00000100:2.0:1787827841.410935:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787827841.410941:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827841/real 1787827841] req@ffff8cb999f38f00 x1874672572489984/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827866 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787827841.410957:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f38f00 x1874672572489984/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827866 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787827841.410963:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787827841.410965:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787827841.410967:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f38f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572489984:192.168.203.129@tcp:38:kworker.0 00010000:00010000:1.0:1787827842.242383:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787827843.266394:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787827844.290371:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787827845.314608:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787827846.338524:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787827846.530411:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827846.530426:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787827846.530432:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827846.530447:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 798 00000100:00080000:3.0:1787827846.530463:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827846.530474:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:3.0:1787827846.530528:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb992315a40 x1874672572490112/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787827846.530567:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827846.530573:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787827846.530575:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827846.530580:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 798 00000100:00080000:3.0:1787827846.530585:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827846.530589:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:3.0:1787827846.530602:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787827846.530618:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb9923161c0 x1874672572490240/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787827846.530672:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827846.530677:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1787827846.530679:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827846.530685:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 798 00000100:00080000:3.0:1787827846.530691:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827846.530697:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:3.0:1787827846.530705:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787827846.530722:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb9923170c0 x1874672572490368/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787827846.530732:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb992315a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572490112:192.168.203.129@tcp:250:kworker.0 00000100:00100000:2.0:1787827846.530940:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb9923161c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572490240:192.168.203.129@tcp:38:kworker.0 00000100:00100000:2.0:1787827846.531035:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb9923170c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572490368:192.168.203.129@tcp:38:kworker.0 00010000:00010000:0.0:1787827847.362552:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787827848.386440:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787827849.410579:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:0.0:1787827849.602564:0:2384:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.129:988 00000400:00000100:0.0:1787827849.602651:0:2384:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.129@tcp at host 192.168.203.129 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:0.0:1787827849.602670:0:2384:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.129@tcp 00000800:00000100:0.0:1787827849.602715:0:2384:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.129@tcp 00000800:00000100:0.0:1787827849.602728:0:2384:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.129@tcp 00000100:00100000:2.0:1787827849.602783:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827846/real 1787827849] req@ffff8cb9923170c0 x1874672572490368/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827871 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787827849.602817:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb9923170c0 x1874672572490368/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827871 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787827849.602833:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787827849.602839:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787827849.602845:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb9923170c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572490368:192.168.203.129@tcp:38:kworker.0 00000100:00100000:2.0:1787827849.602855:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827846/real 1787827849] req@ffff8cb992315a40 x1874672572490112/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827871 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787827849.602872:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb992315a40 x1874672572490112/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827871 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787827849.602901:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787827849.602907:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787827849.602922:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb992315a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572490112:192.168.203.129@tcp:250:kworker.0 00000100:00100000:2.0:1787827849.602931:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827846/real 1787827849] req@ffff8cb9923161c0 x1874672572490240/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827871 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787827849.602949:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb9923161c0 x1874672572490240/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827871 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787827849.602960:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787827849.602964:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787827849.602969:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb9923161c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572490240:192.168.203.129@tcp:38:kworker.0 00010000:00010000:0.0:1787827850.434541:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787827851.458790:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787827851.650389:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787827851.650397:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787827851.650400:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787827851.650409:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 803 00000100:00080000:1.0:1787827851.650418:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787827851.650423:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:1.0:1787827851.650452:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f39e00 x1874672572490496/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787827851.650486:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787827851.650489:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787827851.650490:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787827851.650493:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 803 00000100:00080000:1.0:1787827851.650495:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787827851.650498:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:1.0:1787827851.650505:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787827851.650515:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f38b40 x1874672572490624/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787827851.650541:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1787827851.650547:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f39e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572490496:192.168.203.129@tcp:250:kworker.0 00000100:00080000:1.0:1787827851.650561:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1787827851.650564:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787827851.650569:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 803 00000100:00080000:1.0:1787827851.650574:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787827851.650579:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:1.0:1787827851.650585:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787827851.650614:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f3b0c0 x1874672572490752/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1787827851.650614:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787827851.650658:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827851/real 1787827851] req@ffff8cb999f39e00 x1874672572490496/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827876 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787827851.650679:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f39e00 x1874672572490496/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827876 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787827851.650691:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787827851.650696:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787827851.650700:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f39e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572490496:192.168.203.129@tcp:250:kworker.0 00000100:00100000:2.0:1787827851.650728:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f38b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572490624:192.168.203.129@tcp:38:kworker.0 00000800:00000100:2.0:1787827851.650770:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787827851.650785:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827851/real 1787827851] req@ffff8cb999f38b40 x1874672572490624/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827876 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787827851.650797:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f38b40 x1874672572490624/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827876 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787827851.650805:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787827851.650809:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787827851.650812:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f38b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572490624:192.168.203.129@tcp:38:kworker.0 00000100:00100000:2.0:1787827851.650818:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f3b0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572490752:192.168.203.129@tcp:38:kworker.0 00000800:00000100:2.0:1787827851.650848:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787827851.650860:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827851/real 1787827851] req@ffff8cb999f3b0c0 x1874672572490752/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827876 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787827851.650873:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f3b0c0 x1874672572490752/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827876 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787827851.650881:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787827851.650885:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787827851.650888:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f3b0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572490752:192.168.203.129@tcp:38:kworker.0 00010000:00010000:0.0:1787827852.482429:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787827853.506521:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787827854.530650:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787827855.554790:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787827856.578838:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787827856.770352:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827856.770358:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787827856.770361:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827856.770368:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 808 00000100:00080000:3.0:1787827856.770377:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827856.770382:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:3.0:1787827856.770412:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb992316940 x1874672572490880/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787827856.770436:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827856.770438:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787827856.770440:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827856.770442:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 808 00000100:00080000:3.0:1787827856.770444:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827856.770446:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:3.0:1787827856.770469:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787827856.770479:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb992315e00 x1874672572491008/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787827856.770489:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827856.770491:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1787827856.770493:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827856.770497:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 808 00000100:00080000:3.0:1787827856.770502:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1787827856.770507:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb992316940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572490880:192.168.203.129@tcp:250:kworker.0 00000100:00080000:3.0:1787827856.770507:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:3.0:1787827856.770531:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787827856.770544:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb992314f00 x1874672572491136/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1787827856.770653:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787827856.770741:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827856/real 1787827856] req@ffff8cb992316940 x1874672572490880/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827881 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787827856.770802:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb992316940 x1874672572490880/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827881 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787827856.770814:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787827856.770819:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787827856.770823:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb992316940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572490880:192.168.203.129@tcp:250:kworker.0 00000100:00100000:2.0:1787827856.770855:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb992315e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572491008:192.168.203.129@tcp:38:kworker.0 00000800:00000100:2.0:1787827856.770882:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787827856.770897:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827856/real 1787827856] req@ffff8cb992315e00 x1874672572491008/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827881 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787827856.770915:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb992315e00 x1874672572491008/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827881 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787827856.770927:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787827856.770932:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787827856.770937:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb992315e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572491008:192.168.203.129@tcp:38:kworker.0 00000100:00100000:2.0:1787827856.770947:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb992314f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572491136:192.168.203.129@tcp:38:kworker.0 00000800:00000100:2.0:1787827856.770979:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787827856.770996:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827856/real 1787827856] req@ffff8cb992314f00 x1874672572491136/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827881 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787827856.771007:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb992314f00 x1874672572491136/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827881 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787827856.771028:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787827856.771032:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787827856.771035:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb992314f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572491136:192.168.203.129@tcp:38:kworker.0 00010000:00010000:2.0:1787827857.602654:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787827858.626588:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787827859.650510:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787827860.674388:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787827861.698436:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787827861.890379:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787827861.890389:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787827861.890394:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787827861.890404:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 814 00000100:00080000:0.0:1787827861.890416:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787827861.890424:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:0.0:1787827861.890468:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c6bc00 x1874672572491264/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787827861.890496:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787827861.890499:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787827861.890500:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787827861.890503:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 814 00000100:00080000:0.0:1787827861.890505:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787827861.890508:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:0.0:1787827861.890546:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787827861.890555:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c6bc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572491264:192.168.203.129@tcp:250:kworker.0 00000100:00100000:0.0:1787827861.890564:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c68f00 x1874672572491392/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787827861.890577:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787827861.890579:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1787827861.890582:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787827861.890586:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 814 00000100:00080000:0.0:1787827861.890589:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787827861.890593:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:0.0:1787827861.890597:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787827861.890606:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c69e00 x1874672572491520/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1787827861.890623:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787827861.890665:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827861/real 1787827861] req@ffff8cb999c6bc00 x1874672572491264/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827886 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787827861.890681:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c6bc00 x1874672572491264/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827886 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787827861.890691:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787827861.890695:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787827861.890697:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c6bc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572491264:192.168.203.129@tcp:250:kworker.0 00000100:00100000:2.0:1787827861.890719:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c68f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572491392:192.168.203.129@tcp:38:kworker.0 00000800:00000100:2.0:1787827861.890735:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787827861.890744:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827861/real 1787827861] req@ffff8cb999c68f00 x1874672572491392/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827886 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787827861.890753:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c68f00 x1874672572491392/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827886 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787827861.890759:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787827861.890771:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787827861.890774:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c68f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572491392:192.168.203.129@tcp:38:kworker.0 00000100:00100000:2.0:1787827861.890779:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c69e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572491520:192.168.203.129@tcp:38:kworker.0 00000800:00000100:2.0:1787827861.890794:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787827861.890802:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827861/real 1787827861] req@ffff8cb999c69e00 x1874672572491520/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827886 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787827861.890810:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c69e00 x1874672572491520/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827886 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787827861.890815:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787827861.890818:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787827861.890820:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c69e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572491520:192.168.203.129@tcp:38:kworker.0 00010000:00010000:0.0:1787827862.722541:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787827863.746433:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787827864.770449:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787827865.794439:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787827866.818543:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787827867.010440:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787827867.010455:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787827867.010462:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787827867.010480:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 819 00000100:00080000:0.0:1787827867.010518:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787827867.010532:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:0.0:1787827867.010591:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c6b840 x1874672572491648/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787827867.010656:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787827867.010689:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787827867.010694:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787827867.010704:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 819 00000100:00080000:0.0:1787827867.010723:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787827867.010731:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:3.0:1787827867.010741:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c6b840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572491648:192.168.203.129@tcp:250:kworker.0 00000008:00100000:0.0:1787827867.010752:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787827867.010782:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c69a40 x1874672572491776/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787827867.010803:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787827867.010809:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1787827867.010813:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787827867.010821:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 819 00000100:00080000:0.0:1787827867.010828:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787827867.010835:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:0.0:1787827867.010843:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000800:00000100:3.0:1787827867.010862:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787827867.010868:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c69e00 x1874672572491904/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1787827867.010927:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827867/real 1787827867] req@ffff8cb999c6b840 x1874672572491648/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827892 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827867.010958:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c6b840 x1874672572491648/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827892 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827867.010970:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827867.010974:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:3.0:1787827867.010978:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c6b840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572491648:192.168.203.129@tcp:250:kworker.0 00000100:00100000:3.0:1787827867.011008:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c69a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572491776:192.168.203.129@tcp:38:kworker.0 00000800:00000100:3.0:1787827867.011049:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:3.0:1787827867.011063:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827867/real 1787827867] req@ffff8cb999c69a40 x1874672572491776/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827892 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827867.011076:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c69a40 x1874672572491776/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827892 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827867.011085:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827867.011088:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:3.0:1787827867.011091:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c69a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572491776:192.168.203.129@tcp:38:kworker.0 00000100:00100000:3.0:1787827867.011097:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c69e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572491904:192.168.203.129@tcp:38:kworker.0 00000800:00000100:3.0:1787827867.011115:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:3.0:1787827867.011130:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827867/real 1787827867] req@ffff8cb999c69e00 x1874672572491904/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827892 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827867.011148:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c69e00 x1874672572491904/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827892 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827867.011161:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827867.011166:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:3.0:1787827867.011172:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c69e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572491904:192.168.203.129@tcp:38:kworker.0 00010000:00010000:0.0:1787827867.842426:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787827868.866532:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787827869.890425:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787827870.914530:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787827871.938430:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787827872.130426:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827872.130443:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787827872.130495:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827872.130505:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 824 00000100:00080000:2.0:1787827872.130517:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827872.130525:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:2.0:1787827872.130560:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8d680 x1874672572492032/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787827872.130588:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827872.130606:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787827872.130608:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827872.130614:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 824 00000100:00080000:2.0:1787827872.130619:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827872.130625:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:3.0:1787827872.130644:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8d680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572492032:192.168.203.129@tcp:250:kworker.0 00000008:00100000:2.0:1787827872.130646:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787827872.130681:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8c780 x1874672572492160/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787827872.130706:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827872.130713:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1787827872.130718:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827872.130726:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 824 00000100:00080000:2.0:1787827872.130747:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000800:00000100:3.0:1787827872.130751:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00080000:2.0:1787827872.130782:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:2.0:1787827872.130795:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787827872.130822:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8c000 x1874672572492288/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1787827872.130825:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827872/real 1787827872] req@ffff8cb998a8d680 x1874672572492032/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827897 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827872.130937:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8d680 x1874672572492032/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827897 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827872.130947:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827872.130950:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:3.0:1787827872.130953:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8d680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572492032:192.168.203.129@tcp:250:kworker.0 00000100:00100000:3.0:1787827872.130973:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8c780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572492160:192.168.203.129@tcp:38:kworker.0 00000800:00000100:3.0:1787827872.130989:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:3.0:1787827872.130998:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827872/real 1787827872] req@ffff8cb998a8c780 x1874672572492160/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827897 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827872.131006:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8c780 x1874672572492160/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827897 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827872.131012:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827872.131014:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:3.0:1787827872.131016:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8c780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572492160:192.168.203.129@tcp:38:kworker.0 00000100:00100000:3.0:1787827872.131020:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572492288:192.168.203.129@tcp:38:kworker.0 00000800:00000100:3.0:1787827872.131029:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:3.0:1787827872.131035:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827872/real 1787827872] req@ffff8cb998a8c000 x1874672572492288/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827897 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827872.131042:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8c000 x1874672572492288/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827897 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827872.131068:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827872.131070:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:3.0:1787827872.131073:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572492288:192.168.203.129@tcp:38:kworker.0 00010000:00010000:2.0:1787827872.962519:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787827873.986380:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787827875.010537:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787827876.034357:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787827877.058466:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787827877.250402:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787827877.250411:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787827877.250433:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787827877.250442:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 829 00000100:00080000:1.0:1787827877.250453:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787827877.250460:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:1.0:1787827877.250505:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f392c0 x1874672572492416/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787827877.250532:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787827877.250536:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787827877.250538:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787827877.250541:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 829 00000100:00080000:1.0:1787827877.250544:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787827877.250547:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:1.0:1787827877.250556:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787827877.250595:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f3b480 x1874672572492544/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1787827877.250607:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f392c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572492416:192.168.203.129@tcp:250:kworker.0 00000100:00080000:1.0:1787827877.250610:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787827877.250618:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1787827877.250620:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787827877.250625:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 829 00000100:00080000:1.0:1787827877.250628:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787827877.250632:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:1.0:1787827877.250637:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787827877.250649:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f3a580 x1874672572492672/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1787827877.250691:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:3.0:1787827877.250744:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827877/real 1787827877] req@ffff8cb999f392c0 x1874672572492416/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827902 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827877.250760:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f392c0 x1874672572492416/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827902 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827877.250771:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827877.250774:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:3.0:1787827877.250777:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f392c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572492416:192.168.203.129@tcp:250:kworker.0 00000100:00100000:3.0:1787827877.250800:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f3b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572492544:192.168.203.129@tcp:38:kworker.0 00000800:00000100:3.0:1787827877.250818:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:3.0:1787827877.250828:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827877/real 1787827877] req@ffff8cb999f3b480 x1874672572492544/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827902 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827877.250838:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f3b480 x1874672572492544/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827902 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827877.250845:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827877.250848:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:3.0:1787827877.250850:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f3b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572492544:192.168.203.129@tcp:38:kworker.0 00000100:00100000:3.0:1787827877.250879:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f3a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572492672:192.168.203.129@tcp:38:kworker.0 00000800:00000100:3.0:1787827877.250899:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:3.0:1787827877.250909:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827877/real 1787827877] req@ffff8cb999f3a580 x1874672572492672/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827902 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827877.250920:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f3a580 x1874672572492672/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827902 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827877.250927:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827877.250930:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:3.0:1787827877.250933:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f3a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572492672:192.168.203.129@tcp:38:kworker.0 00010000:00010000:0.0:1787827878.082457:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787827879.106381:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787827880.130439:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787827881.154428:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787827882.178387:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787827882.370341:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827882.370349:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787827882.370353:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827882.370361:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 834 00000100:00080000:3.0:1787827882.370373:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827882.370390:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:3.0:1787827882.370430:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb992317480 x1874672572492800/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787827882.370461:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827882.370465:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787827882.370467:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827882.370470:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 834 00000100:00080000:3.0:1787827882.370474:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827882.370494:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:1.0:1787827882.370495:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb992317480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572492800:192.168.203.129@tcp:250:kworker.0 00000008:00100000:3.0:1787827882.370504:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787827882.370520:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb992314b40 x1874672572492928/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787827882.370529:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827882.370532:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1787827882.370534:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827882.370537:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 834 00000100:00080000:3.0:1787827882.370543:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827882.370545:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:3.0:1787827882.370550:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000800:00000100:1.0:1787827882.370560:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:3.0:1787827882.370565:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb992314780 x1874672572493056/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787827882.370611:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827882/real 1787827882] req@ffff8cb992317480 x1874672572492800/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827907 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787827882.370633:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb992317480 x1874672572492800/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827907 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787827882.370645:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787827882.370649:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787827882.370653:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb992317480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572492800:192.168.203.129@tcp:250:kworker.0 00000100:00100000:1.0:1787827882.370677:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb992314b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572492928:192.168.203.129@tcp:38:kworker.0 00000800:00000100:1.0:1787827882.370699:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787827882.370719:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827882/real 1787827882] req@ffff8cb992314b40 x1874672572492928/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827907 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787827882.370729:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb992314b40 x1874672572492928/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827907 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787827882.370735:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787827882.370737:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787827882.370740:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb992314b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572492928:192.168.203.129@tcp:38:kworker.0 00000100:00100000:1.0:1787827882.370744:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb992314780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572493056:192.168.203.129@tcp:38:kworker.0 00000800:00000100:1.0:1787827882.370757:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787827882.370763:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827882/real 1787827882] req@ffff8cb992314780 x1874672572493056/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827907 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787827882.370771:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb992314780 x1874672572493056/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827907 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787827882.370777:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787827882.370779:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787827882.370781:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb992314780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572493056:192.168.203.129@tcp:38:kworker.0 00010000:00010000:0.0:1787827883.202382:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787827884.226432:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787827885.250393:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787827886.274391:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787827887.298359:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787827887.490338:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787827887.490358:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787827887.490362:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787827887.490372:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 839 00000100:00080000:0.0:1787827887.490383:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787827887.490390:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:0.0:1787827887.490422:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c68f00 x1874672572493184/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787827887.490444:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787827887.490447:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787827887.490449:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787827887.490451:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 839 00000100:00080000:0.0:1787827887.490455:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787827887.490457:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:0.0:1787827887.490465:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787827887.490475:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c6bc00 x1874672572493312/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787827887.490484:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c68f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572493184:192.168.203.129@tcp:250:kworker.0 00000100:00080000:0.0:1787827887.490485:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787827887.490492:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1787827887.490500:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787827887.490504:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 839 00000100:00080000:0.0:1787827887.490507:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787827887.490510:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:0.0:1787827887.490515:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787827887.490524:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c6b480 x1874672572493440/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1787827887.490541:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787827887.490574:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827887/real 1787827887] req@ffff8cb999c68f00 x1874672572493184/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827912 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787827887.490591:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c68f00 x1874672572493184/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827912 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787827887.490606:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787827887.490609:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787827887.490612:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c68f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572493184:192.168.203.129@tcp:250:kworker.0 00000100:00100000:1.0:1787827887.490631:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c6bc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572493312:192.168.203.129@tcp:38:kworker.0 00000800:00000100:1.0:1787827887.490659:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787827887.490674:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827887/real 1787827887] req@ffff8cb999c6bc00 x1874672572493312/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827912 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787827887.490685:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c6bc00 x1874672572493312/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827912 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787827887.490691:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787827887.490694:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787827887.490696:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c6bc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572493312:192.168.203.129@tcp:38:kworker.0 00000100:00100000:1.0:1787827887.490700:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c6b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572493440:192.168.203.129@tcp:38:kworker.0 00000800:00000100:1.0:1787827887.490713:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787827887.490719:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827887/real 1787827887] req@ffff8cb999c6b480 x1874672572493440/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827912 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787827887.490726:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c6b480 x1874672572493440/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827912 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787827887.490732:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787827887.490733:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787827887.490735:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c6b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572493440:192.168.203.129@tcp:38:kworker.0 00010000:00010000:2.0:1787827888.322369:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787827889.346384:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787827890.370421:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787827891.394448:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787827892.418411:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787827892.610367:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827892.610376:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787827892.610379:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827892.610387:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 844 00000100:00080000:3.0:1787827892.610396:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827892.610402:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:3.0:1787827892.610433:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb992316580 x1874672572493568/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787827892.610455:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827892.610459:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787827892.610477:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827892.610482:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 844 00000100:00080000:3.0:1787827892.610487:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827892.610492:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:3.0:1787827892.610503:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787827892.610508:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb992316580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572493568:192.168.203.129@tcp:250:kworker.0 00000100:00100000:3.0:1787827892.610520:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb992314780 x1874672572493696/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787827892.610530:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827892.610532:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1787827892.610534:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827892.610537:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 844 00000100:00080000:3.0:1787827892.610540:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827892.610543:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:3.0:1787827892.610547:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787827892.610555:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb992314b40 x1874672572493824/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1787827892.610573:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787827892.610614:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827892/real 1787827892] req@ffff8cb992316580 x1874672572493568/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827917 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787827892.610632:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb992316580 x1874672572493568/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827917 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787827892.610643:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787827892.610647:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787827892.610652:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb992316580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572493568:192.168.203.129@tcp:250:kworker.0 00000100:00100000:1.0:1787827892.610679:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb992314780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572493696:192.168.203.129@tcp:38:kworker.0 00000800:00000100:1.0:1787827892.610701:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787827892.610713:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827892/real 1787827892] req@ffff8cb992314780 x1874672572493696/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827917 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787827892.610726:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb992314780 x1874672572493696/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827917 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787827892.610734:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787827892.610738:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787827892.610741:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb992314780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572493696:192.168.203.129@tcp:38:kworker.0 00000100:00100000:1.0:1787827892.610747:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb992314b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572493824:192.168.203.129@tcp:38:kworker.0 00000800:00000100:1.0:1787827892.610763:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787827892.610773:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827892/real 1787827892] req@ffff8cb992314b40 x1874672572493824/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827917 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787827892.610793:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb992314b40 x1874672572493824/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827917 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787827892.610803:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787827892.610806:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787827892.610810:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb992314b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572493824:192.168.203.129@tcp:38:kworker.0 00010000:00010000:0.0:1787827893.442409:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787827894.466419:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787827895.490394:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787827896.514391:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787827897.538402:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787827897.730337:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787827897.730347:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787827897.730351:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787827897.730361:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 849 00000100:00080000:1.0:1787827897.730372:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787827897.730377:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:1.0:1787827897.730406:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f383c0 x1874672572493952/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787827897.730428:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787827897.730431:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787827897.730432:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787827897.730435:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 849 00000100:00080000:1.0:1787827897.730438:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787827897.730441:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:1.0:1787827897.730451:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787827897.730459:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f383c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572493952:192.168.203.129@tcp:250:kworker.0 00000100:00100000:1.0:1787827897.730468:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f3a1c0 x1874672572494080/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787827897.730496:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787827897.730500:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1787827897.730502:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787827897.730507:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 849 00000100:00080000:1.0:1787827897.730512:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000800:00000100:3.0:1787827897.730516:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00080000:1.0:1787827897.730519:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:1.0:1787827897.730528:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787827897.730541:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f3a580 x1874672572494208/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1787827897.730553:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827897/real 1787827897] req@ffff8cb999f383c0 x1874672572493952/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827922 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827897.730568:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f383c0 x1874672572493952/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827922 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827897.730576:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827897.730585:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:3.0:1787827897.730588:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f383c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572493952:192.168.203.129@tcp:250:kworker.0 00000100:00100000:3.0:1787827897.730610:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f3a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572494080:192.168.203.129@tcp:38:kworker.0 00000800:00000100:3.0:1787827897.730627:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:3.0:1787827897.730635:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827897/real 1787827897] req@ffff8cb999f3a1c0 x1874672572494080/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827922 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827897.730643:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f3a1c0 x1874672572494080/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827922 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827897.730648:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827897.730651:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:3.0:1787827897.730653:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f3a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572494080:192.168.203.129@tcp:38:kworker.0 00000100:00100000:3.0:1787827897.730656:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f3a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572494208:192.168.203.129@tcp:38:kworker.0 00000800:00000100:3.0:1787827897.730666:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:3.0:1787827897.730672:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827897/real 1787827897] req@ffff8cb999f3a580 x1874672572494208/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827922 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827897.730679:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f3a580 x1874672572494208/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827922 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827897.730684:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827897.730686:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:3.0:1787827897.730687:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f3a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572494208:192.168.203.129@tcp:38:kworker.0 00010000:00010000:0.0:1787827898.562414:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787827899.586411:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787827900.610404:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787827901.634414:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787827902.658380:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787827902.850347:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827902.850367:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787827902.850371:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827902.850378:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 855 00000100:00080000:3.0:1787827902.850386:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827902.850392:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:3.0:1787827902.850419:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb992317480 x1874672572494336/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787827902.850444:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827902.850446:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787827902.850448:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827902.850450:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 855 00000100:00080000:3.0:1787827902.850452:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827902.850454:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:3.0:1787827902.850462:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787827902.850470:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb992314f00 x1874672572494464/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787827902.850478:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827902.850480:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1787827902.850482:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827902.850487:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 855 00000100:00080000:3.0:1787827902.850490:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1787827902.850491:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb992317480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572494336:192.168.203.129@tcp:250:kworker.0 00000100:00080000:3.0:1787827902.850495:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:3.0:1787827902.850501:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787827902.850507:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb992315e00 x1874672572494592/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1787827902.850560:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787827902.850601:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827902/real 1787827902] req@ffff8cb992317480 x1874672572494336/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827927 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787827902.850622:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb992317480 x1874672572494336/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827927 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787827902.850634:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787827902.850638:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787827902.850643:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb992317480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572494336:192.168.203.129@tcp:250:kworker.0 00000100:00100000:1.0:1787827902.850669:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb992314f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572494464:192.168.203.129@tcp:38:kworker.0 00000800:00000100:1.0:1787827902.850696:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787827902.850721:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827902/real 1787827902] req@ffff8cb992314f00 x1874672572494464/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827927 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787827902.850736:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb992314f00 x1874672572494464/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827927 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787827902.850746:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787827902.850749:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787827902.850752:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb992314f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572494464:192.168.203.129@tcp:38:kworker.0 00000100:00100000:1.0:1787827902.850759:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb992315e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572494592:192.168.203.129@tcp:38:kworker.0 00000800:00000100:1.0:1787827902.850783:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787827902.850798:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827902/real 1787827902] req@ffff8cb992315e00 x1874672572494592/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827927 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787827902.850811:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb992315e00 x1874672572494592/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827927 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787827902.850819:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787827902.850823:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787827902.850826:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb992315e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572494592:192.168.203.129@tcp:38:kworker.0 00010000:00010000:1.0:1787827903.682401:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787827904.706392:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787827905.730418:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787827906.754380:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787827907.778385:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787827907.970338:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787827907.970346:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787827907.970349:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787827907.970358:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 860 00000100:00080000:0.0:1787827907.970367:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787827907.970372:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:0.0:1787827907.970407:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c69680 x1874672572494720/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787827907.970454:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787827907.970457:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787827907.970459:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787827907.970462:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 860 00000100:00080000:0.0:1787827907.970465:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787827907.970468:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:0.0:1787827907.970476:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787827907.970486:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c68000 x1874672572494848/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787827907.970494:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787827907.970497:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1787827907.970498:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787827907.970504:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 860 00000100:00100000:1.0:1787827907.970506:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c69680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572494720:192.168.203.129@tcp:250:kworker.0 00000100:00080000:0.0:1787827907.970507:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787827907.970514:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:0.0:1787827907.970518:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787827907.970525:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c6a1c0 x1874672572494976/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1787827907.970566:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787827907.970599:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827907/real 1787827907] req@ffff8cb999c69680 x1874672572494720/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827932 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787827907.970620:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c69680 x1874672572494720/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827932 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787827907.970628:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787827907.970631:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787827907.970634:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c69680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572494720:192.168.203.129@tcp:250:kworker.0 00000100:00100000:1.0:1787827907.970639:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c68000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572494848:192.168.203.129@tcp:38:kworker.0 00000800:00000100:1.0:1787827907.970665:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787827907.970673:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827907/real 1787827907] req@ffff8cb999c68000 x1874672572494848/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827932 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787827907.970681:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c68000 x1874672572494848/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827932 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787827907.970687:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787827907.970689:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787827907.970691:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c68000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572494848:192.168.203.129@tcp:38:kworker.0 00000100:00100000:1.0:1787827907.970710:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c6a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572494976:192.168.203.129@tcp:38:kworker.0 00000800:00000100:1.0:1787827907.970721:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787827907.970727:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827907/real 1787827907] req@ffff8cb999c6a1c0 x1874672572494976/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827932 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787827907.970735:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c6a1c0 x1874672572494976/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827932 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787827907.970749:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787827907.970752:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787827907.970754:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c6a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572494976:192.168.203.129@tcp:38:kworker.0 00010000:00010000:0.0:1787827908.802422:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787827909.826387:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787827910.850391:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787827911.874414:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787827912.898422:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787827913.090354:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787827913.090369:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787827913.090375:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787827913.090386:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 865 00000100:00080000:0.0:1787827913.090420:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787827913.090427:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:0.0:1787827913.090467:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c692c0 x1874672572495104/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787827913.090493:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787827913.090496:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787827913.090497:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787827913.090500:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 865 00000100:00080000:0.0:1787827913.090502:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787827913.090505:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:0.0:1787827913.090515:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787827913.090525:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c6a580 x1874672572495232/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787827913.090534:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787827913.090537:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1787827913.090539:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787827913.090546:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 865 00000100:00100000:1.0:1787827913.090549:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c692c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572495104:192.168.203.129@tcp:250:kworker.0 00000100:00080000:0.0:1787827913.090550:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787827913.090562:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:0.0:1787827913.090570:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787827913.090582:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c6a1c0 x1874672572495360/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787827913.090687:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c6a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572495232:192.168.203.129@tcp:38:kworker.0 00000100:00100000:1.0:1787827913.090726:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c6a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572495360:192.168.203.129@tcp:38:kworker.0 00010000:00010000:1.0:1787827913.922418:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787827914.946377:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787827915.970537:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:1.0:1787827916.162528:0:2381:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.129:988 00000400:00000100:1.0:1787827916.162571:0:2381:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.129@tcp at host 192.168.203.129 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:1.0:1787827916.162585:0:2381:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.129@tcp 00000800:00000100:1.0:1787827916.162613:0:2381:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.129@tcp 00000800:00000100:1.0:1787827916.162619:0:2381:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.129@tcp 00000100:00100000:3.0:1787827916.162659:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827913/real 1787827916] req@ffff8cb999c6a580 x1874672572495232/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827938 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827916.162686:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c6a580 x1874672572495232/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827938 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827916.162700:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827916.162705:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:3.0:1787827916.162711:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c6a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572495232:192.168.203.129@tcp:38:kworker.0 00000100:00100000:3.0:1787827916.162726:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827913/real 1787827916] req@ffff8cb999c6a1c0 x1874672572495360/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827938 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827916.162741:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c6a1c0 x1874672572495360/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827938 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827916.162750:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827916.162754:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:3.0:1787827916.162757:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c6a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572495360:192.168.203.129@tcp:38:kworker.0 00000100:00100000:3.0:1787827916.162765:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827913/real 1787827916] req@ffff8cb999c692c0 x1874672572495104/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827938 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827916.162778:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c692c0 x1874672572495104/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827938 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827916.162787:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827916.162790:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:3.0:1787827916.162808:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c692c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572495104:192.168.203.129@tcp:250:kworker.0 00010000:00010000:1.0:1787827916.994728:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787827918.018460:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787827918.210364:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787827918.210375:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787827918.210379:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787827918.210389:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 870 00000100:00080000:1.0:1787827918.210414:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787827918.210424:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:1.0:1787827918.210460:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f3b480 x1874672572495488/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787827918.210490:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787827918.210495:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787827918.210497:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787827918.210502:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 870 00000100:00080000:1.0:1787827918.210507:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787827918.210513:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:1.0:1787827918.210523:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787827918.210535:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f3b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572495488:192.168.203.129@tcp:250:kworker.0 00000100:00100000:1.0:1787827918.210543:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f392c0 x1874672572495616/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787827918.210557:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787827918.210561:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1787827918.210564:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787827918.210568:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 870 00000100:00080000:1.0:1787827918.210573:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787827918.210577:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:1.0:1787827918.210582:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787827918.210596:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f3b0c0 x1874672572495744/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1787827918.210607:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787827918.210660:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827918/real 1787827918] req@ffff8cb999f3b480 x1874672572495488/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827943 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787827918.210682:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f3b480 x1874672572495488/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827943 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787827918.210694:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787827918.210708:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787827918.210713:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f3b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572495488:192.168.203.129@tcp:250:kworker.0 00000100:00100000:2.0:1787827918.210737:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f392c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572495616:192.168.203.129@tcp:38:kworker.0 00000800:00000100:2.0:1787827918.210761:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787827918.210775:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827918/real 1787827918] req@ffff8cb999f392c0 x1874672572495616/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827943 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787827918.210794:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f392c0 x1874672572495616/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827943 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787827918.210803:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787827918.210807:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787827918.210810:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f392c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572495616:192.168.203.129@tcp:38:kworker.0 00000100:00100000:2.0:1787827918.210817:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f3b0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572495744:192.168.203.129@tcp:38:kworker.0 00000800:00000100:2.0:1787827918.210838:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787827918.210849:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827918/real 1787827918] req@ffff8cb999f3b0c0 x1874672572495744/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827943 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787827918.210878:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f3b0c0 x1874672572495744/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827943 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787827918.210886:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787827918.210890:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787827918.210893:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f3b0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572495744:192.168.203.129@tcp:38:kworker.0 00010000:00010000:1.0:1787827919.042505:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787827920.066464:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787827921.091129:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787827922.115040:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787827923.138849:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787827923.330437:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787827923.330455:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787827923.330463:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787827923.330480:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 875 00000100:00080000:1.0:1787827923.330499:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787827923.330511:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:1.0:1787827923.330577:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f38b40 x1874672572495872/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787827923.330631:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787827923.330636:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787827923.330638:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787827923.330642:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 875 00000100:00080000:1.0:1787827923.330648:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787827923.330653:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:1.0:1787827923.330666:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787827923.330681:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f39e00 x1874672572496000/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787827923.330695:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787827923.330700:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1787827923.330702:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787827923.330707:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 875 00000100:00100000:2.0:1787827923.330727:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f38b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572495872:192.168.203.129@tcp:250:kworker.0 00000100:00080000:1.0:1787827923.330743:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787827923.330752:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:1.0:1787827923.330761:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787827923.330778:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f38f00 x1874672572496128/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1787827923.330806:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787827923.330853:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827923/real 1787827923] req@ffff8cb999f38b40 x1874672572495872/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827948 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787827923.330873:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f38b40 x1874672572495872/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827948 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787827923.330886:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787827923.330891:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787827923.330896:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f38b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572495872:192.168.203.129@tcp:250:kworker.0 00000100:00100000:2.0:1787827923.330905:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f39e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572496000:192.168.203.129@tcp:38:kworker.0 00000800:00000100:2.0:1787827923.330924:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787827923.330943:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827923/real 1787827923] req@ffff8cb999f39e00 x1874672572496000/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827948 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787827923.330955:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f39e00 x1874672572496000/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827948 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787827923.330987:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787827923.330992:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787827923.330996:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f39e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572496000:192.168.203.129@tcp:38:kworker.0 00000100:00100000:2.0:1787827923.331026:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f38f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572496128:192.168.203.129@tcp:38:kworker.0 00000800:00000100:2.0:1787827923.331051:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787827923.331064:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827923/real 1787827923] req@ffff8cb999f38f00 x1874672572496128/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827948 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787827923.331075:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f38f00 x1874672572496128/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827948 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787827923.331083:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787827923.331087:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787827923.331090:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f38f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572496128:192.168.203.129@tcp:38:kworker.0 00010000:00010000:1.0:1787827924.162629:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787827925.186724:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787827926.210517:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787827927.234456:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787827928.258539:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787827928.450414:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827928.450429:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787827928.450436:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827928.450451:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 880 00000100:00080000:2.0:1787827928.450468:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827928.450481:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:2.0:1787827928.450534:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8f480 x1874672572496256/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787827928.450575:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827928.450600:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787827928.450604:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827928.450611:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 880 00000100:00080000:2.0:1787827928.450619:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827928.450625:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:2.0:1787827928.450640:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787827928.450660:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8cb40 x1874672572496384/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1787827928.450672:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572496256:192.168.203.129@tcp:250:kworker.0 00000100:00080000:2.0:1787827928.450696:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827928.450706:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1787827928.450709:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827928.450720:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 880 00000100:00080000:2.0:1787827928.450730:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827928.450740:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:2.0:1787827928.450752:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000800:00000100:3.0:1787827928.450774:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787827928.450782:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8e940 x1874672572496512/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1787827928.450860:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827928/real 1787827928] req@ffff8cb998a8f480 x1874672572496256/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827953 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827928.450892:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8f480 x1874672572496256/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827953 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827928.450907:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827928.450913:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:3.0:1787827928.450918:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572496256:192.168.203.129@tcp:250:kworker.0 00000100:00100000:3.0:1787827928.450949:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8cb40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572496384:192.168.203.129@tcp:38:kworker.0 00000800:00000100:3.0:1787827928.450979:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:3.0:1787827928.451005:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827928/real 1787827928] req@ffff8cb998a8cb40 x1874672572496384/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827953 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827928.451021:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8cb40 x1874672572496384/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827953 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827928.451032:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827928.451036:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:3.0:1787827928.451040:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8cb40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572496384:192.168.203.129@tcp:38:kworker.0 00000100:00100000:3.0:1787827928.451048:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8e940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572496512:192.168.203.129@tcp:38:kworker.0 00000800:00000100:3.0:1787827928.451103:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:3.0:1787827928.451137:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827928/real 1787827928] req@ffff8cb998a8e940 x1874672572496512/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827953 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827928.451154:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8e940 x1874672572496512/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827953 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827928.451165:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827928.451169:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:3.0:1787827928.451173:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8e940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572496512:192.168.203.129@tcp:38:kworker.0 00010000:00010000:1.0:1787827929.282446:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787827930.306520:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787827931.330405:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787827932.354527:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787827933.378568:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787827933.570460:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787827933.570472:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787827933.570477:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787827933.570490:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 885 00000100:00080000:1.0:1787827933.570502:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787827933.570510:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:1.0:1787827933.570553:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f39a40 x1874672572496640/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787827933.570588:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787827933.570593:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787827933.570595:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787827933.570632:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 885 00000100:00080000:1.0:1787827933.570638:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787827933.570649:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:1.0:1787827933.570661:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787827933.570682:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f3b840 x1874672572496768/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1787827933.570693:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f39a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572496640:192.168.203.129@tcp:250:kworker.0 00000100:00080000:1.0:1787827933.570702:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787827933.570717:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1787827933.570721:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787827933.570727:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 885 00000100:00080000:1.0:1787827933.570730:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787827933.570733:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:1.0:1787827933.570738:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787827933.570754:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f38f00 x1874672572496896/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1787827933.570859:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:3.0:1787827933.570981:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827933/real 1787827933] req@ffff8cb999f39a40 x1874672572496640/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827958 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827933.571011:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f39a40 x1874672572496640/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827958 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827933.571056:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827933.571062:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:3.0:1787827933.571069:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f39a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572496640:192.168.203.129@tcp:250:kworker.0 00000100:00100000:3.0:1787827933.571111:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f3b840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572496768:192.168.203.129@tcp:38:kworker.0 00000800:00000100:3.0:1787827933.571150:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:3.0:1787827933.571169:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827933/real 1787827933] req@ffff8cb999f3b840 x1874672572496768/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827958 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827933.571186:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f3b840 x1874672572496768/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827958 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827933.571225:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827933.571230:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:3.0:1787827933.571235:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f3b840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572496768:192.168.203.129@tcp:38:kworker.0 00000100:00100000:3.0:1787827933.571244:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f38f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572496896:192.168.203.129@tcp:38:kworker.0 00000800:00000100:3.0:1787827933.571321:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:3.0:1787827933.571338:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827933/real 1787827933] req@ffff8cb999f38f00 x1874672572496896/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827958 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827933.571353:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f38f00 x1874672572496896/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827958 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827933.571364:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827933.571368:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:3.0:1787827933.571372:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f38f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572496896:192.168.203.129@tcp:38:kworker.0 00010000:00010000:1.0:1787827934.402449:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787827935.426544:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787827936.450375:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787827937.474410:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787827938.498495:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787827938.690341:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787827938.690355:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787827938.690378:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787827938.690394:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 890 00000100:00080000:1.0:1787827938.690411:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787827938.690423:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:1.0:1787827938.690487:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f39e00 x1874672572497024/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787827938.690545:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787827938.690552:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787827938.690556:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787827938.690564:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 890 00000100:00080000:1.0:1787827938.690573:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787827938.690583:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:1.0:1787827938.690619:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787827938.690623:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f39e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572497024:192.168.203.129@tcp:250:kworker.0 00000100:00100000:1.0:1787827938.690646:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f38b40 x1874672572497152/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787827938.690666:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787827938.690671:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1787827938.690674:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787827938.690679:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 890 00000100:00080000:1.0:1787827938.690686:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787827938.690692:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:1.0:1787827938.690699:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000800:00000100:3.0:1787827938.690714:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787827938.690721:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f3b0c0 x1874672572497280/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1787827938.690779:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827938/real 1787827938] req@ffff8cb999f39e00 x1874672572497024/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827963 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827938.690820:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f39e00 x1874672572497024/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827963 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827938.690832:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827938.690835:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:3.0:1787827938.690851:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f39e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572497024:192.168.203.129@tcp:250:kworker.0 00000100:00100000:3.0:1787827938.690879:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f38b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572497152:192.168.203.129@tcp:38:kworker.0 00000800:00000100:3.0:1787827938.690946:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:3.0:1787827938.690982:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827938/real 1787827938] req@ffff8cb999f38b40 x1874672572497152/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827963 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827938.691015:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f38b40 x1874672572497152/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827963 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827938.691023:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827938.691027:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:3.0:1787827938.691030:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f38b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572497152:192.168.203.129@tcp:38:kworker.0 00000100:00100000:3.0:1787827938.691037:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f3b0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572497280:192.168.203.129@tcp:38:kworker.0 00000800:00000100:3.0:1787827938.691057:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:3.0:1787827938.691067:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827938/real 1787827938] req@ffff8cb999f3b0c0 x1874672572497280/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827963 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827938.691078:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f3b0c0 x1874672572497280/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827963 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827938.691086:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827938.691089:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:3.0:1787827938.691092:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f3b0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572497280:192.168.203.129@tcp:38:kworker.0 00010000:00010000:1.0:1787827939.522413:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787827940.546448:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787827941.570475:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787827942.594575:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787827943.618597:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787827943.810442:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827943.810456:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787827943.810484:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827943.810498:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 895 00000100:00080000:3.0:1787827943.810512:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827943.810521:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:3.0:1787827943.810578:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb992316940 x1874672572497408/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787827943.810619:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827943.810625:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787827943.810629:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827943.810634:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 895 00000100:00080000:3.0:1787827943.810640:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827943.810646:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:3.0:1787827943.810662:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787827943.810682:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb9923161c0 x1874672572497536/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787827943.810716:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1787827943.810718:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb992316940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572497408:192.168.203.129@tcp:250:kworker.0 00000100:00080000:3.0:1787827943.810722:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1787827943.810725:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827943.810732:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 895 00000100:00080000:3.0:1787827943.810738:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827943.810745:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:3.0:1787827943.810754:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787827943.810770:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb992315a40 x1874672572497664/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1787827943.810853:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787827943.810921:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827943/real 1787827943] req@ffff8cb992316940 x1874672572497408/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827968 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787827943.810944:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb992316940 x1874672572497408/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827968 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787827943.811007:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787827943.811016:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787827943.811023:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb992316940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572497408:192.168.203.129@tcp:250:kworker.0 00000100:00100000:2.0:1787827943.811059:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb9923161c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572497536:192.168.203.129@tcp:38:kworker.0 00000800:00000100:2.0:1787827943.811090:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787827943.811105:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827943/real 1787827943] req@ffff8cb9923161c0 x1874672572497536/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827968 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787827943.811119:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb9923161c0 x1874672572497536/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827968 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787827943.811128:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787827943.811131:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787827943.811135:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb9923161c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572497536:192.168.203.129@tcp:38:kworker.0 00000100:00100000:2.0:1787827943.811141:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb992315a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572497664:192.168.203.129@tcp:38:kworker.0 00000800:00000100:2.0:1787827943.811159:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787827943.811168:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827943/real 1787827943] req@ffff8cb992315a40 x1874672572497664/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827968 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787827943.811180:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb992315a40 x1874672572497664/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827968 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787827943.811188:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787827943.811191:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787827943.811194:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb992315a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572497664:192.168.203.129@tcp:38:kworker.0 00010000:00010000:1.0:1787827944.642492:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787827945.666408:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787827946.690501:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787827947.714533:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787827948.738507:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787827948.930328:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827948.930339:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787827948.930344:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827948.930355:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 901 00000100:00080000:3.0:1787827948.930366:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827948.930372:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:3.0:1787827948.930404:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb9923170c0 x1874672572497792/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787827948.930432:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827948.930435:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787827948.930437:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827948.930439:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 901 00000100:00080000:3.0:1787827948.930442:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827948.930444:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:3.0:1787827948.930452:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787827948.930461:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb992315a40 x1874672572497920/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787827948.930468:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827948.930471:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1787827948.930473:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827948.930475:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 901 00000100:00080000:3.0:1787827948.930478:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827948.930481:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:3.0:1787827948.930498:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787827948.930501:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb9923170c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572497792:192.168.203.129@tcp:250:kworker.0 00000100:00100000:3.0:1787827948.930508:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb9923161c0 x1874672572498048/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1787827948.930566:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787827948.930616:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827948/real 1787827948] req@ffff8cb9923170c0 x1874672572497792/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827973 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787827948.930631:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb9923170c0 x1874672572497792/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827973 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787827948.930640:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787827948.930644:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787827948.930649:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb9923170c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572497792:192.168.203.129@tcp:250:kworker.0 00000100:00100000:2.0:1787827948.930655:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb992315a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572497920:192.168.203.129@tcp:38:kworker.0 00000800:00000100:2.0:1787827948.930680:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787827948.930691:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827948/real 1787827948] req@ffff8cb992315a40 x1874672572497920/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827973 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787827948.930699:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb992315a40 x1874672572497920/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827973 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787827948.930705:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787827948.930709:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787827948.930713:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb992315a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572497920:192.168.203.129@tcp:38:kworker.0 00000100:00100000:2.0:1787827948.930755:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb9923161c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572498048:192.168.203.129@tcp:38:kworker.0 00000800:00000100:2.0:1787827948.930775:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787827948.930784:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827948/real 1787827948] req@ffff8cb9923161c0 x1874672572498048/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827973 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787827948.930800:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb9923161c0 x1874672572498048/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827973 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787827948.930807:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787827948.930809:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787827948.930811:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb9923161c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572498048:192.168.203.129@tcp:38:kworker.0 00010000:00010000:0.0:1787827949.762444:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787827950.786385:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787827951.810389:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787827952.834388:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787827953.858481:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787827954.050401:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827954.050416:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787827954.050422:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827954.050454:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 906 00000100:00080000:3.0:1787827954.050470:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827954.050482:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:3.0:1787827954.050548:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb992316940 x1874672572498176/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787827954.050591:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827954.050598:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787827954.050602:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827954.050610:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 906 00000100:00080000:3.0:1787827954.050618:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827954.050625:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:3.0:1787827954.050656:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787827954.050682:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb992316940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572498176:192.168.203.129@tcp:250:kworker.0 00000100:00100000:3.0:1787827954.050690:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb992315e00 x1874672572498304/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787827954.050708:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827954.050713:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1787827954.050716:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827954.050722:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 906 00000100:00080000:3.0:1787827954.050783:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827954.050792:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000800:00000100:2.0:1787827954.050820:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000008:00100000:3.0:1787827954.050827:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787827954.050849:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb992314f00 x1874672572498432/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787827954.050907:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827954/real 1787827954] req@ffff8cb992316940 x1874672572498176/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827979 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787827954.050964:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb992316940 x1874672572498176/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827979 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787827954.050979:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787827954.050985:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787827954.050992:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb992316940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572498176:192.168.203.129@tcp:250:kworker.0 00000100:00100000:2.0:1787827954.051022:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb992315e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572498304:192.168.203.129@tcp:38:kworker.0 00000800:00000100:2.0:1787827954.051055:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787827954.051075:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827954/real 1787827954] req@ffff8cb992315e00 x1874672572498304/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827979 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787827954.051092:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb992315e00 x1874672572498304/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827979 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787827954.051123:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787827954.051129:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787827954.051134:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb992315e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572498304:192.168.203.129@tcp:38:kworker.0 00000100:00100000:2.0:1787827954.051142:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb992314f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572498432:192.168.203.129@tcp:38:kworker.0 00000800:00000100:2.0:1787827954.051192:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787827954.051215:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827954/real 1787827954] req@ffff8cb992314f00 x1874672572498432/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827979 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787827954.051239:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb992314f00 x1874672572498432/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827979 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787827954.051256:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787827954.051262:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787827954.051268:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb992314f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572498432:192.168.203.129@tcp:38:kworker.0 00010000:00010000:1.0:1787827954.882427:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787827955.906414:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787827956.930534:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787827957.954472:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787827958.978518:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787827959.170402:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787827959.170417:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787827959.170423:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787827959.170438:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 911 00000100:00080000:1.0:1787827959.170452:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787827959.170460:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:1.0:1787827959.170501:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f392c0 x1874672572498560/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787827959.170535:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787827959.170539:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787827959.170542:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787827959.170545:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 911 00000100:00080000:1.0:1787827959.170550:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787827959.170553:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:1.0:1787827959.170563:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787827959.170576:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f3b480 x1874672572498688/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787827959.170608:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787827959.170613:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1787827959.170615:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787827959.170621:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 911 00000100:00080000:1.0:1787827959.170626:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787827959.170632:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:1.0:1787827959.170638:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787827959.170680:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f3a580 x1874672572498816/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787827959.170687:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f392c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572498560:192.168.203.129@tcp:250:kworker.0 00000800:00000100:2.0:1787827959.170782:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787827959.170830:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827959/real 1787827959] req@ffff8cb999f392c0 x1874672572498560/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827984 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787827959.170881:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f392c0 x1874672572498560/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827984 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787827959.170895:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787827959.170900:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787827959.170917:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f392c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572498560:192.168.203.129@tcp:250:kworker.0 00000100:00100000:2.0:1787827959.170927:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f3b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572498688:192.168.203.129@tcp:38:kworker.0 00000800:00000100:2.0:1787827959.170958:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787827959.170974:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827959/real 1787827959] req@ffff8cb999f3b480 x1874672572498688/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827984 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787827959.170988:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f3b480 x1874672572498688/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827984 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787827959.170998:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787827959.171002:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787827959.171006:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f3b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572498688:192.168.203.129@tcp:38:kworker.0 00000100:00100000:2.0:1787827959.171034:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f3a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572498816:192.168.203.129@tcp:38:kworker.0 00000800:00000100:2.0:1787827959.171053:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787827959.171065:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827959/real 1787827959] req@ffff8cb999f3a580 x1874672572498816/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827984 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787827959.171087:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f3a580 x1874672572498816/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827984 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787827959.171097:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787827959.171100:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787827959.171104:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f3a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572498816:192.168.203.129@tcp:38:kworker.0 00010000:00010000:1.0:1787827960.002443:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787827961.026482:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787827962.050448:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787827963.074425:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787827964.098524:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787827964.290496:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827964.290507:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787827964.290523:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827964.290536:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 916 00000100:00080000:3.0:1787827964.290553:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827964.290564:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:3.0:1787827964.290629:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb992317480 x1874672572498944/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787827964.290693:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827964.290698:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787827964.290701:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827964.290705:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 916 00000100:00080000:3.0:1787827964.290720:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827964.290725:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:3.0:1787827964.290738:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787827964.290756:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb992314b40 x1874672572499072/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787827964.290772:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1787827964.290780:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb992317480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572498944:192.168.203.129@tcp:250:kworker.0 00000100:00080000:3.0:1787827964.290849:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1787827964.290864:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827964.290872:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 916 00000100:00080000:3.0:1787827964.290880:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827964.290886:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:3.0:1787827964.290895:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787827964.290911:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb992314780 x1874672572499200/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1787827964.290929:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787827964.290999:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827964/real 1787827964] req@ffff8cb992317480 x1874672572498944/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827989 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787827964.291018:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb992317480 x1874672572498944/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827989 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787827964.291030:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787827964.291035:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787827964.291039:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb992317480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572498944:192.168.203.129@tcp:250:kworker.0 00000100:00100000:2.0:1787827964.291065:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb992314b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572499072:192.168.203.129@tcp:38:kworker.0 00000800:00000100:2.0:1787827964.291105:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787827964.291117:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827964/real 1787827964] req@ffff8cb992314b40 x1874672572499072/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827989 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787827964.291128:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb992314b40 x1874672572499072/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827989 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787827964.291136:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787827964.291139:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787827964.291142:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb992314b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572499072:192.168.203.129@tcp:38:kworker.0 00000100:00100000:2.0:1787827964.291147:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb992314780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572499200:192.168.203.129@tcp:38:kworker.0 00000800:00000100:2.0:1787827964.291161:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787827964.291169:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827964/real 1787827964] req@ffff8cb992314780 x1874672572499200/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827989 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787827964.291180:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb992314780 x1874672572499200/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827989 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787827964.291200:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787827964.291203:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787827964.291206:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb992314780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572499200:192.168.203.129@tcp:38:kworker.0 00010000:00010000:0.0:1787827965.122451:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787827966.146457:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787827967.170521:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787827968.194481:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787827969.218466:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787827969.410426:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827969.410439:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787827969.410445:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827969.410457:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 921 00000100:00080000:2.0:1787827969.410470:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827969.410479:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:2.0:1787827969.410524:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8ed00 x1874672572499328/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787827969.410578:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827969.410586:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787827969.410591:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827969.410598:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 921 00000100:00080000:2.0:1787827969.410606:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827969.410614:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:2.0:1787827969.410629:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787827969.410629:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8ed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572499328:192.168.203.129@tcp:250:kworker.0 00000100:00100000:2.0:1787827969.410668:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8f840 x1874672572499456/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787827969.410687:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827969.410692:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1787827969.410711:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827969.410717:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 921 00000100:00080000:2.0:1787827969.410724:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827969.410730:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000800:00000100:3.0:1787827969.410734:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000008:00100000:2.0:1787827969.410739:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787827969.410759:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8e940 x1874672572499584/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1787827969.410832:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827969/real 1787827969] req@ffff8cb998a8ed00 x1874672572499328/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827994 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827969.410864:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8ed00 x1874672572499328/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827994 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827969.410883:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827969.410889:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:3.0:1787827969.410895:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8ed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572499328:192.168.203.129@tcp:250:kworker.0 00000100:00100000:3.0:1787827969.410938:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8f840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572499456:192.168.203.129@tcp:38:kworker.0 00000800:00000100:3.0:1787827969.410974:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:3.0:1787827969.410991:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827969/real 1787827969] req@ffff8cb998a8f840 x1874672572499456/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827994 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827969.411006:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8f840 x1874672572499456/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827994 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827969.411028:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827969.411032:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:3.0:1787827969.411037:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8f840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572499456:192.168.203.129@tcp:38:kworker.0 00000100:00100000:3.0:1787827969.411045:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8e940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572499584:192.168.203.129@tcp:38:kworker.0 00000800:00000100:3.0:1787827969.411070:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:3.0:1787827969.411083:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827969/real 1787827969] req@ffff8cb998a8e940 x1874672572499584/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827994 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827969.411096:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8e940 x1874672572499584/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827994 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827969.411104:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827969.411107:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:3.0:1787827969.411111:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8e940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572499584:192.168.203.129@tcp:38:kworker.0 00010000:00010000:1.0:1787827970.242467:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787827971.266454:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787827972.290474:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787827973.314478:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787827974.338492:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787827974.530451:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787827974.530464:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787827974.530470:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787827974.530510:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 926 00000100:00080000:1.0:1787827974.530522:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787827974.530530:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:1.0:1787827974.530599:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f3a1c0 x1874672572499712/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787827974.530644:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787827974.530649:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787827974.530651:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787827974.530656:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 926 00000100:00080000:1.0:1787827974.530661:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787827974.530668:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:1.0:1787827974.530681:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787827974.530698:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f383c0 x1874672572499840/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1787827974.530710:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f3a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572499712:192.168.203.129@tcp:250:kworker.0 00000100:00080000:1.0:1787827974.530717:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787827974.530724:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1787827974.530726:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787827974.530732:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 926 00000100:00080000:1.0:1787827974.530757:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787827974.530763:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:1.0:1787827974.530770:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787827974.530788:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f3a580 x1874672572499968/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1787827974.530814:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:3.0:1787827974.530872:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827974/real 1787827974] req@ffff8cb999f3a1c0 x1874672572499712/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827999 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827974.530894:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f3a1c0 x1874672572499712/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827999 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827974.530908:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827974.530914:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:3.0:1787827974.530930:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f3a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572499712:192.168.203.129@tcp:250:kworker.0 00000100:00100000:3.0:1787827974.530960:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f383c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572499840:192.168.203.129@tcp:38:kworker.0 00000800:00000100:3.0:1787827974.530989:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:3.0:1787827974.531004:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827974/real 1787827974] req@ffff8cb999f383c0 x1874672572499840/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827999 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827974.531017:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f383c0 x1874672572499840/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827999 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827974.531026:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827974.531030:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:3.0:1787827974.531033:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f383c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572499840:192.168.203.129@tcp:38:kworker.0 00000100:00100000:3.0:1787827974.531039:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f3a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572499968:192.168.203.129@tcp:38:kworker.0 00000800:00000100:3.0:1787827974.531057:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:3.0:1787827974.531066:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827974/real 1787827974] req@ffff8cb999f3a580 x1874672572499968/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827999 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827974.531078:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f3a580 x1874672572499968/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827999 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827974.531087:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827974.531090:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:3.0:1787827974.531093:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f3a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572499968:192.168.203.129@tcp:38:kworker.0 00010000:00010000:1.0:1787827975.362452:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787827976.386486:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787827977.410533:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787827978.434472:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787827979.458502:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787827979.650396:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787827979.650411:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787827979.650418:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787827979.650435:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 931 00000100:00080000:1.0:1787827979.650451:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787827979.650462:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:1.0:1787827979.650536:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f3b480 x1874672572500096/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787827979.650579:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787827979.650585:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787827979.650588:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787827979.650593:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 931 00000100:00080000:1.0:1787827979.650597:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787827979.650602:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:1.0:1787827979.650618:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787827979.650634:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f392c0 x1874672572500224/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787827979.650649:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787827979.650654:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1787827979.650657:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1787827979.650666:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f3b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572500096:192.168.203.129@tcp:250:kworker.0 00000100:00080000:1.0:1787827979.650667:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 931 00000100:00080000:1.0:1787827979.650677:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787827979.650686:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:1.0:1787827979.650693:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787827979.650776:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f3b0c0 x1874672572500352/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1787827979.650933:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f392c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572500224:192.168.203.129@tcp:38:kworker.0 00000100:00100000:3.0:1787827979.651015:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f3b0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572500352:192.168.203.129@tcp:38:kworker.0 00010000:00010000:0.0:1787827980.482477:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787827981.506665:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787827982.530819:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:0.0:1787827982.722924:0:2382:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.129:988 00000400:00000100:0.0:1787827982.723002:0:2382:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.129@tcp at host 192.168.203.129 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:0.0:1787827982.723033:0:2382:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.129@tcp 00000800:00000100:0.0:1787827982.723093:0:2382:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.129@tcp 00000800:00000100:0.0:1787827982.723107:0:2382:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.129@tcp 00000100:00100000:3.0:1787827982.723198:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827979/real 1787827982] req@ffff8cb999f392c0 x1874672572500224/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828004 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827982.723370:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f392c0 x1874672572500224/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828004 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827982.723401:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827982.723417:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:3.0:1787827982.723429:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f392c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572500224:192.168.203.129@tcp:38:kworker.0 00000100:00100000:3.0:1787827982.723447:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827979/real 1787827982] req@ffff8cb999f3b0c0 x1874672572500352/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828004 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827982.723476:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f3b0c0 x1874672572500352/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828004 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827982.723508:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827982.723516:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:3.0:1787827982.723523:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f3b0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572500352:192.168.203.129@tcp:38:kworker.0 00000100:00100000:3.0:1787827982.723539:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827979/real 1787827982] req@ffff8cb999f3b480 x1874672572500096/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828004 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827982.723565:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f3b480 x1874672572500096/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828004 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827982.724142:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827982.724157:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:3.0:1787827982.724186:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f3b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572500096:192.168.203.129@tcp:250:kworker.0 00010000:00010000:2.0:1787827983.555201:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787827984.578444:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787827984.770342:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787827984.770353:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787827984.770357:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787827984.770367:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 936 00000100:00080000:1.0:1787827984.770379:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787827984.770387:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:1.0:1787827984.770438:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f38b40 x1874672572500480/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787827984.770465:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787827984.770468:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787827984.770470:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787827984.770473:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 936 00000100:00080000:1.0:1787827984.770476:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787827984.770480:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:1.0:1787827984.770489:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787827984.770501:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f39e00 x1874672572500608/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1787827984.770509:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f38b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572500480:192.168.203.129@tcp:250:kworker.0 00000100:00080000:1.0:1787827984.770514:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787827984.770518:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1787827984.770552:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787827984.770558:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 936 00000100:00080000:1.0:1787827984.770563:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787827984.770567:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:1.0:1787827984.770573:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787827984.770587:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f38f00 x1874672572500736/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1787827984.770614:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:3.0:1787827984.770655:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827984/real 1787827984] req@ffff8cb999f38b40 x1874672572500480/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828009 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827984.770709:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f38b40 x1874672572500480/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828009 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827984.770726:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827984.770731:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:3.0:1787827984.770737:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f38b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572500480:192.168.203.129@tcp:250:kworker.0 00000100:00100000:3.0:1787827984.770770:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f39e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572500608:192.168.203.129@tcp:38:kworker.0 00000800:00000100:3.0:1787827984.770804:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:3.0:1787827984.770823:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827984/real 1787827984] req@ffff8cb999f39e00 x1874672572500608/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828009 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827984.770858:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f39e00 x1874672572500608/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828009 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827984.770868:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827984.770872:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:3.0:1787827984.770877:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f39e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572500608:192.168.203.129@tcp:38:kworker.0 00000100:00100000:3.0:1787827984.770884:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f38f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572500736:192.168.203.129@tcp:38:kworker.0 00000800:00000100:3.0:1787827984.770910:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:3.0:1787827984.770939:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827984/real 1787827984] req@ffff8cb999f38f00 x1874672572500736/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828009 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827984.770955:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f38f00 x1874672572500736/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828009 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827984.770974:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827984.770979:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:3.0:1787827984.770983:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f38f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572500736:192.168.203.129@tcp:38:kworker.0 00010000:00010000:0.0:1787827985.602546:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787827986.626451:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787827987.650561:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787827988.674713:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787827989.698510:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787827989.890417:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787827989.890430:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787827989.890436:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787827989.890449:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 942 00000100:00080000:1.0:1787827989.890462:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787827989.890472:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:1.0:1787827989.890522:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f3b840 x1874672572500864/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787827989.890573:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787827989.890579:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787827989.890582:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787827989.890590:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 942 00000100:00080000:1.0:1787827989.890594:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787827989.890599:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:1.0:1787827989.890612:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787827989.890658:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f39a40 x1874672572500992/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787827989.890707:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f3b840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572500864:192.168.203.129@tcp:250:kworker.0 00000100:00080000:1.0:1787827989.890707:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787827989.890721:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1787827989.890728:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787827989.890741:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 942 00000100:00080000:1.0:1787827989.890749:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787827989.890760:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:1.0:1787827989.890771:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787827989.890800:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f38f00 x1874672572501120/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1787827989.890876:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787827989.890963:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827989/real 1787827989] req@ffff8cb999f3b840 x1874672572500864/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828014 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827989.890994:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f3b840 x1874672572500864/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828014 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827989.891011:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827989.891031:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787827989.891038:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f3b840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572500864:192.168.203.129@tcp:250:kworker.0 00000100:00100000:0.0:1787827989.891106:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f39a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572500992:192.168.203.129@tcp:38:kworker.0 00000800:00000100:0.0:1787827989.891148:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787827989.891167:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827989/real 1787827989] req@ffff8cb999f39a40 x1874672572500992/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828014 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827989.891198:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f39a40 x1874672572500992/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828014 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827989.891210:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827989.891214:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787827989.891219:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f39a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572500992:192.168.203.129@tcp:38:kworker.0 00000100:00100000:0.0:1787827989.891227:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f38f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572501120:192.168.203.129@tcp:38:kworker.0 00000800:00000100:0.0:1787827989.891256:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787827989.891341:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827989/real 1787827989] req@ffff8cb999f38f00 x1874672572501120/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828014 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827989.891373:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f38f00 x1874672572501120/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828014 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827989.891391:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827989.891401:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787827989.891419:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f38f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572501120:192.168.203.129@tcp:38:kworker.0 00010000:00010000:1.0:1787827990.722717:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787827991.746702:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787827992.770405:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787827993.794507:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787827994.818448:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787827995.010387:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787827995.010404:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787827995.010411:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787827995.010428:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 947 00000100:00080000:1.0:1787827995.010459:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787827995.010470:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:1.0:1787827995.010520:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f39e00 x1874672572501248/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787827995.010558:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787827995.010563:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787827995.010565:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787827995.010570:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 947 00000100:00080000:1.0:1787827995.010619:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787827995.010631:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:0.0:1787827995.010633:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f39e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572501248:192.168.203.129@tcp:250:kworker.0 00000008:00100000:1.0:1787827995.010646:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787827995.010671:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f38b40 x1874672572501376/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787827995.010690:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787827995.010695:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1787827995.010697:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787827995.010702:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 947 00000100:00080000:1.0:1787827995.010710:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787827995.010714:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:1.0:1787827995.010720:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000800:00000100:0.0:1787827995.010731:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787827995.010740:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f3b480 x1874672572501504/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787827995.010822:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827995/real 1787827995] req@ffff8cb999f39e00 x1874672572501248/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828020 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827995.010843:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f39e00 x1874672572501248/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828020 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827995.010855:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827995.010859:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787827995.010864:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f39e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572501248:192.168.203.129@tcp:250:kworker.0 00000100:00100000:0.0:1787827995.010912:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f38b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572501376:192.168.203.129@tcp:38:kworker.0 00000800:00000100:0.0:1787827995.010944:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787827995.010960:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827995/real 1787827995] req@ffff8cb999f38b40 x1874672572501376/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828020 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827995.010975:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f38b40 x1874672572501376/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828020 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827995.010986:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827995.010990:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787827995.010994:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f38b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572501376:192.168.203.129@tcp:38:kworker.0 00000100:00100000:0.0:1787827995.011001:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f3b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572501504:192.168.203.129@tcp:38:kworker.0 00000800:00000100:0.0:1787827995.011020:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787827995.011049:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827995/real 1787827995] req@ffff8cb999f3b480 x1874672572501504/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828020 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827995.011064:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f3b480 x1874672572501504/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828020 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827995.011105:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827995.011113:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787827995.011117:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f3b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572501504:192.168.203.129@tcp:38:kworker.0 00010000:00010000:0.0:1787827995.842473:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787827996.866649:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787827997.890485:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787827998.914507:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787827999.938467:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787828000.130371:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828000.130385:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787828000.130390:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828000.130404:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 952 00000100:00080000:1.0:1787828000.130418:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828000.130427:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:1.0:1787828000.130472:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f3b0c0 x1874672572501632/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828000.130522:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828000.130525:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787828000.130526:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828000.130529:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 952 00000100:00080000:1.0:1787828000.130533:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828000.130536:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:1.0:1787828000.130546:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787828000.130556:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f392c0 x1874672572501760/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787828000.130581:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f3b0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572501632:192.168.203.129@tcp:250:kworker.0 00000100:00080000:1.0:1787828000.130588:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828000.130592:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1787828000.130593:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828000.130597:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 952 00000100:00080000:1.0:1787828000.130601:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828000.130607:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:1.0:1787828000.130611:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787828000.130623:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f3a580 x1874672572501888/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1787828000.130648:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787828000.130686:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828000/real 1787828000] req@ffff8cb999f3b0c0 x1874672572501632/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828025 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828000.130700:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f3b0c0 x1874672572501632/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828025 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828000.130710:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828000.130721:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787828000.130724:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f3b0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572501632:192.168.203.129@tcp:250:kworker.0 00000100:00100000:0.0:1787828000.130745:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f392c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572501760:192.168.203.129@tcp:38:kworker.0 00000800:00000100:0.0:1787828000.130761:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787828000.130770:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828000/real 1787828000] req@ffff8cb999f392c0 x1874672572501760/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828025 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828000.130797:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f392c0 x1874672572501760/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828025 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828000.130803:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828000.130805:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787828000.130807:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f392c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572501760:192.168.203.129@tcp:38:kworker.0 00000100:00100000:0.0:1787828000.130812:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f3a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572501888:192.168.203.129@tcp:38:kworker.0 00000800:00000100:0.0:1787828000.130826:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787828000.130832:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828000/real 1787828000] req@ffff8cb999f3a580 x1874672572501888/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828025 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828000.130840:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f3a580 x1874672572501888/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828025 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828000.130846:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828000.130848:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787828000.130849:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f3a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572501888:192.168.203.129@tcp:38:kworker.0 00010000:00010000:0.0:1787828000.962500:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787828001.986483:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787828003.010445:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787828004.034509:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787828005.058492:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787828005.250433:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828005.250446:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787828005.250451:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828005.250464:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 957 00000100:00080000:2.0:1787828005.250476:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828005.250485:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:2.0:1787828005.250560:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8cb40 x1874672572502016/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828005.250600:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828005.250605:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787828005.250608:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828005.250613:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 957 00000100:00080000:2.0:1787828005.250617:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828005.250622:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:2.0:1787828005.250636:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787828005.250677:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8cb40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572502016:192.168.203.129@tcp:250:kworker.0 00000100:00100000:2.0:1787828005.250695:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8f480 x1874672572502144/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828005.250714:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828005.250719:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1787828005.250722:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828005.250727:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 957 00000100:00080000:2.0:1787828005.250735:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828005.250740:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:2.0:1787828005.250748:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000800:00000100:0.0:1787828005.250761:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787828005.250770:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8c000 x1874672572502272/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787828005.250833:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828005/real 1787828005] req@ffff8cb998a8cb40 x1874672572502016/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828030 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828005.250898:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8cb40 x1874672572502016/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828030 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828005.250920:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828005.250927:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787828005.250932:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8cb40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572502016:192.168.203.129@tcp:250:kworker.0 00000100:00100000:0.0:1787828005.250965:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572502144:192.168.203.129@tcp:38:kworker.0 00000800:00000100:0.0:1787828005.251009:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787828005.251036:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828005/real 1787828005] req@ffff8cb998a8f480 x1874672572502144/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828030 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828005.251065:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8f480 x1874672572502144/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828030 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828005.251081:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828005.251086:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787828005.251090:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572502144:192.168.203.129@tcp:38:kworker.0 00000100:00100000:0.0:1787828005.251099:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572502272:192.168.203.129@tcp:38:kworker.0 00000800:00000100:0.0:1787828005.251129:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787828005.251144:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828005/real 1787828005] req@ffff8cb998a8c000 x1874672572502272/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828030 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828005.251160:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8c000 x1874672572502272/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828030 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828005.251187:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828005.251192:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787828005.251197:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572502272:192.168.203.129@tcp:38:kworker.0 00010000:00010000:1.0:1787828006.082484:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787828007.106543:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787828008.130553:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787828009.154625:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787828010.178576:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787828010.370415:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828010.370423:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787828010.370442:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828010.370451:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 962 00000100:00080000:1.0:1787828010.370462:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828010.370469:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:1.0:1787828010.370515:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f383c0 x1874672572502400/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828010.370540:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828010.370543:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787828010.370544:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828010.370547:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 962 00000100:00080000:1.0:1787828010.370550:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828010.370552:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:1.0:1787828010.370560:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787828010.370570:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f3a1c0 x1874672572502528/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828010.370578:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828010.370581:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1787828010.370582:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828010.370587:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 962 00000100:00100000:0.0:1787828010.370589:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f383c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572502400:192.168.203.129@tcp:250:kworker.0 00000100:00080000:1.0:1787828010.370590:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828010.370596:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:1.0:1787828010.370602:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787828010.370611:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f3a580 x1874672572502656/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1787828010.370649:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787828010.370682:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828010/real 1787828010] req@ffff8cb999f383c0 x1874672572502400/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828035 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828010.370695:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f383c0 x1874672572502400/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828035 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828010.370713:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828010.370716:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787828010.370719:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f383c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572502400:192.168.203.129@tcp:250:kworker.0 00000100:00100000:0.0:1787828010.370751:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f3a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572502528:192.168.203.129@tcp:38:kworker.0 00000800:00000100:0.0:1787828010.370766:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787828010.370774:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828010/real 1787828010] req@ffff8cb999f3a1c0 x1874672572502528/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828035 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828010.370782:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f3a1c0 x1874672572502528/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828035 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828010.370808:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828010.370810:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787828010.370813:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f3a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572502528:192.168.203.129@tcp:38:kworker.0 00000100:00100000:0.0:1787828010.370817:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f3a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572502656:192.168.203.129@tcp:38:kworker.0 00000800:00000100:0.0:1787828010.370842:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787828010.370850:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828010/real 1787828010] req@ffff8cb999f3a580 x1874672572502656/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828035 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828010.370857:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f3a580 x1874672572502656/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828035 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828010.370863:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828010.370865:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787828010.370867:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f3a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572502656:192.168.203.129@tcp:38:kworker.0 00010000:00010000:0.0:1787828011.202544:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787828012.226533:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787828013.250530:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787828014.274532:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787828015.298496:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787828015.490351:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828015.490364:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787828015.490369:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828015.490380:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 967 00000100:00080000:1.0:1787828015.490391:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828015.490398:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:1.0:1787828015.490431:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f392c0 x1874672572502784/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828015.490470:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828015.490474:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787828015.490476:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828015.490479:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 967 00000100:00080000:1.0:1787828015.490483:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828015.490487:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:1.0:1787828015.490509:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787828015.490520:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f392c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572502784:192.168.203.129@tcp:250:kworker.0 00000100:00100000:1.0:1787828015.490525:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f3b0c0 x1874672572502912/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828015.490537:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828015.490540:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1787828015.490542:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828015.490545:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 967 00000100:00080000:1.0:1787828015.490590:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000800:00000100:0.0:1787828015.490600:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00080000:1.0:1787828015.490601:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:1.0:1787828015.490610:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787828015.490626:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f3b480 x1874672572503040/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787828015.490658:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828015/real 1787828015] req@ffff8cb999f392c0 x1874672572502784/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828040 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828015.490676:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f392c0 x1874672572502784/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828040 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828015.490700:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828015.490704:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787828015.490708:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f392c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572502784:192.168.203.129@tcp:250:kworker.0 00000100:00100000:0.0:1787828015.490731:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f3b0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572502912:192.168.203.129@tcp:38:kworker.0 00000800:00000100:0.0:1787828015.490753:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787828015.490765:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828015/real 1787828015] req@ffff8cb999f3b0c0 x1874672572502912/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828040 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828015.490776:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f3b0c0 x1874672572502912/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828040 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828015.490783:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828015.490786:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787828015.490789:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f3b0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572502912:192.168.203.129@tcp:38:kworker.0 00000100:00100000:0.0:1787828015.490793:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f3b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572503040:192.168.203.129@tcp:38:kworker.0 00000800:00000100:0.0:1787828015.490807:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787828015.490814:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828015/real 1787828015] req@ffff8cb999f3b480 x1874672572503040/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828040 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828015.490824:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f3b480 x1874672572503040/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828040 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828015.490830:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828015.490833:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787828015.490835:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f3b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572503040:192.168.203.129@tcp:38:kworker.0 00010000:00010000:0.0:1787828016.322391:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787828017.346413:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787828018.370390:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787828019.394443:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787828020.418561:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787828020.610458:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828020.610472:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787828020.610478:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828020.610490:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 972 00000100:00080000:0.0:1787828020.610503:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828020.610512:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:0.0:1787828020.610554:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c69680 x1874672572503168/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828020.610593:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828020.610598:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787828020.610601:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828020.610605:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 972 00000100:00080000:0.0:1787828020.610610:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828020.610615:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:0.0:1787828020.610628:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787828020.610651:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c6b480 x1874672572503296/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1787828020.610668:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c69680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572503168:192.168.203.129@tcp:250:kworker.0 00000100:00080000:0.0:1787828020.610682:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828020.610692:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1787828020.610696:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828020.610722:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 972 00000100:00080000:0.0:1787828020.610735:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828020.610744:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:0.0:1787828020.610756:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000800:00000100:3.0:1787828020.610776:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787828020.610785:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c6bc00 x1874672572503424/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1787828020.610863:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828020/real 1787828020] req@ffff8cb999c69680 x1874672572503168/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828045 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828020.610893:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c69680 x1874672572503168/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828045 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828020.610909:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828020.610915:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:3.0:1787828020.610920:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c69680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572503168:192.168.203.129@tcp:250:kworker.0 00000100:00100000:3.0:1787828020.610950:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c6b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572503296:192.168.203.129@tcp:38:kworker.0 00000800:00000100:3.0:1787828020.610982:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:3.0:1787828020.610999:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828020/real 1787828020] req@ffff8cb999c6b480 x1874672572503296/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828045 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828020.611015:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c6b480 x1874672572503296/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828045 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828020.611060:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828020.611065:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:3.0:1787828020.611070:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c6b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572503296:192.168.203.129@tcp:38:kworker.0 00000100:00100000:3.0:1787828020.611079:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c6bc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572503424:192.168.203.129@tcp:38:kworker.0 00000800:00000100:3.0:1787828020.611109:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:3.0:1787828020.611124:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828020/real 1787828020] req@ffff8cb999c6bc00 x1874672572503424/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828045 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828020.611139:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c6bc00 x1874672572503424/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828045 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828020.611149:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828020.611153:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:3.0:1787828020.611157:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c6bc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572503424:192.168.203.129@tcp:38:kworker.0 00010000:00010000:1.0:1787828021.442487:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787828022.466390:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787828023.490480:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787828024.514515:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787828025.538491:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787828025.730424:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828025.730436:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787828025.730441:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828025.730453:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 977 00000100:00080000:0.0:1787828025.730464:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828025.730493:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:0.0:1787828025.730564:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c68f00 x1874672572503552/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828025.730634:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828025.730641:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787828025.730646:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828025.730656:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 977 00000100:00080000:0.0:1787828025.730682:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828025.730688:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:0.0:1787828025.730703:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787828025.730706:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c68f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572503552:192.168.203.129@tcp:250:kworker.0 00000100:00100000:0.0:1787828025.730750:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c69e00 x1874672572503680/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828025.730765:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828025.730770:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1787828025.730772:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828025.730778:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 977 00000100:00080000:0.0:1787828025.730784:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828025.730792:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000800:00000100:2.0:1787828025.730793:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000008:00100000:0.0:1787828025.730822:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787828025.730880:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828025/real 1787828025] req@ffff8cb999c68f00 x1874672572503552/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828050 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828025.730882:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c69a40 x1874672572503808/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787828025.730911:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c68f00 x1874672572503552/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828050 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828025.730921:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828025.730925:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787828025.730947:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c68f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572503552:192.168.203.129@tcp:250:kworker.0 00000100:00100000:2.0:1787828025.730982:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c69e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572503680:192.168.203.129@tcp:38:kworker.0 00000800:00000100:2.0:1787828025.731011:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787828025.731043:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828025/real 1787828025] req@ffff8cb999c69e00 x1874672572503680/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828050 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828025.731058:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c69e00 x1874672572503680/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828050 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828025.731070:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828025.731075:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787828025.731097:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c69e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572503680:192.168.203.129@tcp:38:kworker.0 00000100:00100000:2.0:1787828025.731105:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c69a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572503808:192.168.203.129@tcp:38:kworker.0 00000800:00000100:2.0:1787828025.731131:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787828025.731164:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828025/real 1787828025] req@ffff8cb999c69a40 x1874672572503808/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828050 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828025.731180:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c69a40 x1874672572503808/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828050 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828025.731188:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828025.731191:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787828025.731193:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c69a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572503808:192.168.203.129@tcp:38:kworker.0 00010000:00010000:0.0:1787828026.562392:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787828027.586567:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787828028.610525:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787828029.634719:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787828030.658518:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787828030.850532:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828030.850551:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787828030.850561:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828030.850582:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 983 00000100:00080000:1.0:1787828030.850603:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828030.850619:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:1.0:1787828030.850708:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f38b40 x1874672572503936/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828030.850770:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828030.850775:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787828030.850778:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828030.850784:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 983 00000100:00080000:1.0:1787828030.850791:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828030.850799:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:1.0:1787828030.850815:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787828030.850837:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f38b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572503936:192.168.203.129@tcp:250:kworker.0 00000100:00100000:1.0:1787828030.850883:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f39e00 x1874672572504064/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828030.850904:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828030.850909:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1787828030.850913:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828030.850919:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 983 00000100:00080000:1.0:1787828030.850926:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000800:00000100:2.0:1787828030.850927:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00080000:1.0:1787828030.850939:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:1.0:1787828030.850947:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787828030.850965:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f38f00 x1874672572504192/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787828030.850982:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828030/real 1787828030] req@ffff8cb999f38b40 x1874672572503936/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828055 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828030.851003:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f38b40 x1874672572503936/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828055 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828030.851029:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828030.851033:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787828030.851037:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f38b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572503936:192.168.203.129@tcp:250:kworker.0 00000100:00100000:2.0:1787828030.851063:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f39e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572504064:192.168.203.129@tcp:38:kworker.0 00000800:00000100:2.0:1787828030.851086:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787828030.851097:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828030/real 1787828030] req@ffff8cb999f39e00 x1874672572504064/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828055 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828030.851108:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f39e00 x1874672572504064/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828055 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828030.851115:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828030.851118:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787828030.851121:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f39e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572504064:192.168.203.129@tcp:38:kworker.0 00000100:00100000:2.0:1787828030.851125:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f38f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572504192:192.168.203.129@tcp:38:kworker.0 00000800:00000100:2.0:1787828030.851138:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787828030.851177:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828030/real 1787828030] req@ffff8cb999f38f00 x1874672572504192/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828055 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828030.851194:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f38f00 x1874672572504192/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828055 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828030.851204:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828030.851209:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787828030.851213:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f38f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572504192:192.168.203.129@tcp:38:kworker.0 00010000:00010000:0.0:1787828031.682500:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787828032.706567:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787828033.730405:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787828034.754520:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787828035.778486:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787828035.970423:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828035.970437:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787828035.970442:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828035.970455:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 988 00000100:00080000:1.0:1787828035.970469:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828035.970478:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:1.0:1787828035.970528:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f39a40 x1874672572504320/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828035.970560:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828035.970564:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787828035.970566:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828035.970569:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 988 00000100:00080000:1.0:1787828035.970595:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828035.970604:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:1.0:1787828035.970620:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787828035.970636:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f39a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572504320:192.168.203.129@tcp:250:kworker.0 00000100:00100000:1.0:1787828035.970646:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f3b840 x1874672572504448/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828035.970665:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828035.970669:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1787828035.970671:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828035.970675:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 988 00000100:00080000:1.0:1787828035.970681:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828035.970685:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:1.0:1787828035.970691:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787828035.970716:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f38f00 x1874672572504576/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1787828035.970728:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787828035.970783:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828035/real 1787828035] req@ffff8cb999f39a40 x1874672572504320/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828060 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828035.970810:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f39a40 x1874672572504320/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828060 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828035.970822:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828035.970826:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787828035.970830:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f39a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572504320:192.168.203.129@tcp:250:kworker.0 00000100:00100000:2.0:1787828035.970855:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f3b840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572504448:192.168.203.129@tcp:38:kworker.0 00000800:00000100:2.0:1787828035.970877:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787828035.970889:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828035/real 1787828035] req@ffff8cb999f3b840 x1874672572504448/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828060 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828035.970900:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f3b840 x1874672572504448/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828060 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828035.970908:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828035.970911:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787828035.970914:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f3b840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572504448:192.168.203.129@tcp:38:kworker.0 00000100:00100000:2.0:1787828035.970920:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f38f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572504576:192.168.203.129@tcp:38:kworker.0 00000800:00000100:2.0:1787828035.970934:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787828035.970942:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828035/real 1787828035] req@ffff8cb999f38f00 x1874672572504576/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828060 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828035.970967:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f38f00 x1874672572504576/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828060 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828035.970975:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828035.970978:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787828035.970981:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f38f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572504576:192.168.203.129@tcp:38:kworker.0 00010000:00010000:1.0:1787828036.802551:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787828037.826576:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787828038.850515:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787828039.874531:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787828040.898525:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787828041.090377:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828041.090389:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787828041.090394:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828041.090407:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 993 00000100:00080000:1.0:1787828041.090420:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828041.090431:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:1.0:1787828041.090479:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f39e00 x1874672572504704/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828041.090541:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828041.090546:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787828041.090549:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828041.090554:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 993 00000100:00080000:1.0:1787828041.090559:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828041.090564:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:1.0:1787828041.090577:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787828041.090595:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f38b40 x1874672572504832/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828041.090610:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828041.090615:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1787828041.090618:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828041.090623:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 993 00000100:00080000:1.0:1787828041.090631:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1787828041.090633:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f39e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572504704:192.168.203.129@tcp:250:kworker.0 00000100:00080000:1.0:1787828041.090640:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:1.0:1787828041.090654:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787828041.090669:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f3b480 x1874672572504960/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1787828041.090762:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787828041.090836:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828041/real 1787828041] req@ffff8cb999f39e00 x1874672572504704/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828066 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828041.090867:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f39e00 x1874672572504704/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828066 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828041.090883:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828041.090888:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787828041.090894:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f39e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572504704:192.168.203.129@tcp:250:kworker.0 00000100:00100000:2.0:1787828041.090904:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f38b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572504832:192.168.203.129@tcp:38:kworker.0 00000800:00000100:2.0:1787828041.090935:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787828041.090952:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828041/real 1787828041] req@ffff8cb999f38b40 x1874672572504832/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828066 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828041.090976:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f38b40 x1874672572504832/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828066 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828041.091021:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828041.091028:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787828041.091036:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f38b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572504832:192.168.203.129@tcp:38:kworker.0 00000100:00100000:2.0:1787828041.091087:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f3b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572504960:192.168.203.129@tcp:38:kworker.0 00000800:00000100:2.0:1787828041.091132:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787828041.091156:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828041/real 1787828041] req@ffff8cb999f3b480 x1874672572504960/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828066 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828041.091175:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f3b480 x1874672572504960/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828066 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828041.091186:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828041.091190:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787828041.091194:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f3b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572504960:192.168.203.129@tcp:38:kworker.0 00010000:00010000:0.0:1787828041.922540:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787828042.946402:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787828043.970444:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787828044.994494:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787828046.018517:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787828046.210433:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828046.210446:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787828046.210452:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828046.210465:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 998 00000100:00080000:2.0:1787828046.210477:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828046.210486:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:2.0:1787828046.210544:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8c780 x1874672572505088/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828046.210584:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828046.210607:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787828046.210611:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828046.210618:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 998 00000100:00080000:2.0:1787828046.210625:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828046.210631:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:2.0:1787828046.210645:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787828046.210657:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8c780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572505088:192.168.203.129@tcp:250:kworker.0 00000100:00100000:2.0:1787828046.210669:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8d680 x1874672572505216/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828046.210698:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828046.210703:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1787828046.210706:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828046.210713:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 998 00000100:00080000:2.0:1787828046.210729:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828046.210733:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:2.0:1787828046.210740:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787828046.210756:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8e580 x1874672572505344/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1787828046.210816:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8d680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572505216:192.168.203.129@tcp:38:kworker.0 00000100:00100000:3.0:1787828046.210891:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8e580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572505344:192.168.203.129@tcp:38:kworker.0 00010000:00010000:0.0:1787828047.042612:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787828048.066596:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787828049.091397:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:1.0:1787828049.282712:0:2383:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.129:988 00000400:00000100:1.0:1787828049.282769:0:2383:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.129@tcp at host 192.168.203.129 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:1.0:1787828049.282807:0:2383:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.129@tcp 00000800:00000100:1.0:1787828049.282861:0:2383:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.129@tcp 00000800:00000100:1.0:1787828049.282869:0:2383:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.129@tcp 00000100:00100000:3.0:1787828049.282984:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828046/real 1787828049] req@ffff8cb998a8d680 x1874672572505216/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828071 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828049.283033:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8d680 x1874672572505216/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828071 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828049.283093:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828049.283113:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:3.0:1787828049.283121:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8d680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572505216:192.168.203.129@tcp:38:kworker.0 00000100:00100000:3.0:1787828049.283134:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828046/real 1787828049] req@ffff8cb998a8e580 x1874672572505344/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828071 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828049.283159:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8e580 x1874672572505344/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828071 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828049.283172:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828049.283178:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:3.0:1787828049.283184:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8e580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572505344:192.168.203.129@tcp:38:kworker.0 00000100:00100000:3.0:1787828049.283196:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828046/real 1787828049] req@ffff8cb998a8c780 x1874672572505088/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828071 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828049.283242:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8c780 x1874672572505088/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828071 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828049.283296:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828049.283304:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:3.0:1787828049.283315:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8c780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572505088:192.168.203.129@tcp:250:kworker.0 00010000:00010000:1.0:1787828050.114805:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787828051.139061:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787828051.330397:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828051.330416:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787828051.330424:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828051.330443:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 1003 00000100:00080000:2.0:1787828051.330462:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828051.330477:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:2.0:1787828051.330540:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8cf00 x1874672572505472/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828051.330624:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828051.330629:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787828051.330632:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828051.330638:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 1003 00000100:00080000:2.0:1787828051.330644:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828051.330650:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:2.0:1787828051.330666:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787828051.330695:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8f0c0 x1874672572505600/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1787828051.330732:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8cf00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572505472:192.168.203.129@tcp:250:kworker.0 00000100:00080000:2.0:1787828051.330733:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828051.330754:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1787828051.330758:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828051.330767:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 1003 00000100:00080000:2.0:1787828051.330775:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828051.330784:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:2.0:1787828051.330795:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787828051.330824:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8e1c0 x1874672572505728/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1787828051.330886:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:3.0:1787828051.330962:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828051/real 1787828051] req@ffff8cb998a8cf00 x1874672572505472/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828076 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828051.331016:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8cf00 x1874672572505472/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828076 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828051.331038:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828051.331048:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:3.0:1787828051.331055:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8cf00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572505472:192.168.203.129@tcp:250:kworker.0 00000100:00100000:3.0:1787828051.331103:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8f0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572505600:192.168.203.129@tcp:38:kworker.0 00000800:00000100:3.0:1787828051.331153:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:3.0:1787828051.331180:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828051/real 1787828051] req@ffff8cb998a8f0c0 x1874672572505600/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828076 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828051.331231:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8f0c0 x1874672572505600/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828076 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828051.331250:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828051.331258:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:3.0:1787828051.331314:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8f0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572505600:192.168.203.129@tcp:38:kworker.0 00000100:00100000:3.0:1787828051.331328:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8e1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572505728:192.168.203.129@tcp:38:kworker.0 00000800:00000100:3.0:1787828051.331374:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:3.0:1787828051.331399:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828051/real 1787828051] req@ffff8cb998a8e1c0 x1874672572505728/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828076 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828051.331424:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8e1c0 x1874672572505728/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828076 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828051.331470:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828051.331477:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:3.0:1787828051.331484:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8e1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572505728:192.168.203.129@tcp:38:kworker.0 00010000:00010000:1.0:1787828052.162753:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787828053.187048:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787828054.210725:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787828055.234887:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787828056.258493:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787828056.450430:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828056.450483:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787828056.450489:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828056.450503:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 1008 00000100:00080000:0.0:1787828056.450547:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828056.450560:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:0.0:1787828056.450626:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c6b840 x1874672572505856/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828056.450670:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828056.450675:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787828056.450677:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828056.450682:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 1008 00000100:00080000:0.0:1787828056.450688:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828056.450693:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:0.0:1787828056.450706:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787828056.450744:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c69a40 x1874672572505984/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828056.450758:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828056.450765:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00100000:3.0:1787828056.450765:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c6b840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572505856:192.168.203.129@tcp:250:kworker.0 00000100:00080000:0.0:1787828056.450770:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828056.450790:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 1008 00000100:00080000:0.0:1787828056.450795:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828056.450799:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:0.0:1787828056.450804:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787828056.450827:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c69e00 x1874672572506112/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1787828056.450889:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:3.0:1787828056.450957:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828056/real 1787828056] req@ffff8cb999c6b840 x1874672572505856/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828081 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828056.450987:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c6b840 x1874672572505856/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828081 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828056.451014:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828056.451021:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:3.0:1787828056.451028:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c6b840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572505856:192.168.203.129@tcp:250:kworker.0 00000100:00100000:3.0:1787828056.451081:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c69a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572505984:192.168.203.129@tcp:38:kworker.0 00000800:00000100:3.0:1787828056.451114:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:3.0:1787828056.451130:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828056/real 1787828056] req@ffff8cb999c69a40 x1874672572505984/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828081 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828056.451144:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c69a40 x1874672572505984/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828081 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828056.451154:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828056.451158:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:3.0:1787828056.451162:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c69a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572505984:192.168.203.129@tcp:38:kworker.0 00000100:00100000:3.0:1787828056.451204:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c69e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572506112:192.168.203.129@tcp:38:kworker.0 00000800:00000100:3.0:1787828056.451231:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:3.0:1787828056.451246:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828056/real 1787828056] req@ffff8cb999c69e00 x1874672572506112/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828081 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828056.451297:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c69e00 x1874672572506112/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828081 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828056.451307:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828056.451311:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:3.0:1787828056.451315:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c69e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572506112:192.168.203.129@tcp:38:kworker.0 00010000:00010000:1.0:1787828057.282476:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787828058.306552:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787828059.330445:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787828060.354460:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787828061.378420:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787828061.570395:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828061.570407:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787828061.570413:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828061.570427:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 1013 00000100:00080000:0.0:1787828061.570440:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828061.570448:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:0.0:1787828061.570494:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c68f00 x1874672572506240/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828061.570526:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828061.570531:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787828061.570535:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828061.570540:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 1013 00000100:00080000:0.0:1787828061.570545:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828061.570550:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:0.0:1787828061.570561:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787828061.570577:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c68f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572506240:192.168.203.129@tcp:250:kworker.0 00000100:00100000:0.0:1787828061.570583:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c6bc00 x1874672572506368/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828061.570604:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828061.570608:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1787828061.570610:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828061.570614:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 1013 00000100:00080000:0.0:1787828061.570619:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828061.570623:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:0.0:1787828061.570643:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787828061.570662:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c6b480 x1874672572506496/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1787828061.570703:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787828061.570767:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828061/real 1787828061] req@ffff8cb999c68f00 x1874672572506240/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828086 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828061.570814:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c68f00 x1874672572506240/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828086 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828061.570828:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828061.570833:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787828061.570838:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c68f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572506240:192.168.203.129@tcp:250:kworker.0 00000100:00100000:2.0:1787828061.570870:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c6bc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572506368:192.168.203.129@tcp:38:kworker.0 00000800:00000100:2.0:1787828061.570901:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787828061.570917:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828061/real 1787828061] req@ffff8cb999c6bc00 x1874672572506368/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828086 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828061.570931:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c6bc00 x1874672572506368/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828086 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828061.570941:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828061.570946:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787828061.570949:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c6bc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572506368:192.168.203.129@tcp:38:kworker.0 00000100:00100000:2.0:1787828061.570956:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c6b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572506496:192.168.203.129@tcp:38:kworker.0 00000800:00000100:2.0:1787828061.570978:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787828061.570989:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828061/real 1787828061] req@ffff8cb999c6b480 x1874672572506496/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828086 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828061.571002:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c6b480 x1874672572506496/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828086 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828061.571028:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828061.571032:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787828061.571036:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c6b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572506496:192.168.203.129@tcp:38:kworker.0 00010000:00010000:1.0:1787828062.402415:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787828063.426522:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787828064.450509:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787828065.474586:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787828066.498539:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787828066.690457:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828066.690473:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787828066.690480:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828066.690496:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 1018 00000100:00080000:2.0:1787828066.690513:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828066.690523:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:2.0:1787828066.690590:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8da40 x1874672572506624/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828066.690653:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828066.690662:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787828066.690666:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828066.690675:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 1018 00000100:00080000:2.0:1787828066.690682:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828066.690691:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:2.0:1787828066.690711:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787828066.690737:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8fc00 x1874672572506752/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828066.690754:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828066.690759:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00100000:3.0:1787828066.690782:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8da40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572506624:192.168.203.129@tcp:250:kworker.0 00000100:00080000:2.0:1787828066.690794:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828066.690804:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 1018 00000100:00080000:2.0:1787828066.690810:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828066.690816:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:2.0:1787828066.690826:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787828066.690847:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8d2c0 x1874672572506880/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1787828066.690907:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:3.0:1787828066.691007:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828066/real 1787828066] req@ffff8cb998a8da40 x1874672572506624/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828091 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828066.691046:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8da40 x1874672572506624/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828091 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828066.691066:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828066.691073:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:3.0:1787828066.691083:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8da40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572506624:192.168.203.129@tcp:250:kworker.0 00000100:00100000:3.0:1787828066.691137:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8fc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572506752:192.168.203.129@tcp:38:kworker.0 00000800:00000100:3.0:1787828066.691187:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:3.0:1787828066.691215:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828066/real 1787828066] req@ffff8cb998a8fc00 x1874672572506752/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828091 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828066.691316:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8fc00 x1874672572506752/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828091 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828066.691330:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828066.691334:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:3.0:1787828066.691339:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8fc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572506752:192.168.203.129@tcp:38:kworker.0 00000100:00100000:3.0:1787828066.691378:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8d2c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572506880:192.168.203.129@tcp:38:kworker.0 00000800:00000100:3.0:1787828066.691412:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:3.0:1787828066.691428:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828066/real 1787828066] req@ffff8cb998a8d2c0 x1874672572506880/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828091 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828066.691444:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8d2c0 x1874672572506880/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828091 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828066.691454:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828066.691458:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:3.0:1787828066.691462:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8d2c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572506880:192.168.203.129@tcp:38:kworker.0 00010000:00010000:1.0:1787828067.522492:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787828068.546540:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787828069.570469:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787828070.594443:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787828071.618478:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787828071.810491:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828071.810505:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787828071.810511:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828071.810523:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 1023 00000100:00080000:1.0:1787828071.810537:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828071.810566:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:1.0:1787828071.810610:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f3b0c0 x1874672572507008/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828071.810649:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828071.810654:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787828071.810657:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828071.810662:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 1023 00000100:00080000:1.0:1787828071.810668:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828071.810672:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:1.0:1787828071.810684:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787828071.810717:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f392c0 x1874672572507136/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828071.810739:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828071.810747:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1787828071.810751:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828071.810760:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 1023 00000100:00080000:1.0:1787828071.810768:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828071.810778:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:1.0:1787828071.810789:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787828071.810795:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f3b0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572507008:192.168.203.129@tcp:250:kworker.0 00000100:00100000:1.0:1787828071.810818:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f3a580 x1874672572507264/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1787828071.810904:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:3.0:1787828071.810972:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828071/real 1787828071] req@ffff8cb999f3b0c0 x1874672572507008/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828096 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828071.811000:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f3b0c0 x1874672572507008/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828096 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828071.811036:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828071.811042:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:3.0:1787828071.811048:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f3b0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572507008:192.168.203.129@tcp:250:kworker.0 00000100:00100000:3.0:1787828071.811058:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f392c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572507136:192.168.203.129@tcp:38:kworker.0 00000800:00000100:3.0:1787828071.811122:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:3.0:1787828071.811139:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828071/real 1787828071] req@ffff8cb999f392c0 x1874672572507136/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828096 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828071.811155:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f392c0 x1874672572507136/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828096 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828071.811166:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828071.811170:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:3.0:1787828071.811174:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f392c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572507136:192.168.203.129@tcp:38:kworker.0 00000100:00100000:3.0:1787828071.811205:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f3a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572507264:192.168.203.129@tcp:38:kworker.0 00000800:00000100:3.0:1787828071.811226:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:3.0:1787828071.811239:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828071/real 1787828071] req@ffff8cb999f3a580 x1874672572507264/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828096 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828071.811253:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f3a580 x1874672572507264/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828096 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828071.811308:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828071.811313:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:3.0:1787828071.811317:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f3a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572507264:192.168.203.129@tcp:38:kworker.0 00010000:00010000:1.0:1787828072.642501:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787828073.666401:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787828074.690525:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787828075.714406:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787828076.738408:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787828076.930436:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828076.930448:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787828076.930454:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828076.930466:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 1029 00000100:00080000:0.0:1787828076.930479:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828076.930487:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:0.0:1787828076.930549:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c69680 x1874672572507392/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828076.930600:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828076.930604:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787828076.930607:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828076.930611:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 1029 00000100:00080000:0.0:1787828076.930615:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828076.930619:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:0.0:1787828076.930630:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787828076.930645:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c68000 x1874672572507520/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828076.930659:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828076.930663:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1787828076.930666:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1787828076.930674:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c69680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572507392:192.168.203.129@tcp:250:kworker.0 00000100:00080000:0.0:1787828076.930679:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 1029 00000100:00080000:0.0:1787828076.930714:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828076.930723:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:0.0:1787828076.930733:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000800:00000100:3.0:1787828076.930754:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787828076.930763:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c692c0 x1874672572507648/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1787828076.930833:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828076/real 1787828076] req@ffff8cb999c69680 x1874672572507392/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828101 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828076.930859:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c69680 x1874672572507392/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828101 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828076.930873:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828076.930879:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:3.0:1787828076.930885:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c69680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572507392:192.168.203.129@tcp:250:kworker.0 00000100:00100000:3.0:1787828076.930911:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c68000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572507520:192.168.203.129@tcp:38:kworker.0 00000800:00000100:3.0:1787828076.930935:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:3.0:1787828076.930948:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828076/real 1787828076] req@ffff8cb999c68000 x1874672572507520/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828101 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828076.930961:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c68000 x1874672572507520/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828101 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828076.930970:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828076.930974:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:3.0:1787828076.930978:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c68000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572507520:192.168.203.129@tcp:38:kworker.0 00000100:00100000:3.0:1787828076.930984:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c692c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572507648:192.168.203.129@tcp:38:kworker.0 00000800:00000100:3.0:1787828076.931000:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:3.0:1787828076.931009:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828076/real 1787828076] req@ffff8cb999c692c0 x1874672572507648/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828101 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828076.931089:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c692c0 x1874672572507648/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828101 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828076.931107:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828076.931113:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:3.0:1787828076.931121:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c692c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572507648:192.168.203.129@tcp:38:kworker.0 00010000:00010000:1.0:1787828077.762483:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787828078.786507:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787828079.810487:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787828080.834494:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787828081.858529:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787828082.050428:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828082.050440:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787828082.050445:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828082.050457:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 1034 00000100:00080000:1.0:1787828082.050468:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828082.050476:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:1.0:1787828082.050520:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f3a1c0 x1874672572507776/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828082.050573:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828082.050578:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787828082.050581:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828082.050586:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 1034 00000100:00080000:1.0:1787828082.050591:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828082.050596:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:1.0:1787828082.050609:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787828082.050654:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f383c0 x1874672572507904/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1787828082.050670:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f3a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572507776:192.168.203.129@tcp:250:kworker.0 00000100:00080000:1.0:1787828082.050671:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828082.050695:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1787828082.050698:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828082.050704:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 1034 00000100:00080000:1.0:1787828082.050710:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828082.050714:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:1.0:1787828082.050722:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787828082.050740:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f3a580 x1874672572508032/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1787828082.050760:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:3.0:1787828082.050825:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828082/real 1787828082] req@ffff8cb999f3a1c0 x1874672572507776/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828107 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828082.050851:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f3a1c0 x1874672572507776/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828107 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828082.050864:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828082.050868:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:3.0:1787828082.050873:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f3a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572507776:192.168.203.129@tcp:250:kworker.0 00000100:00100000:3.0:1787828082.050900:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f383c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572507904:192.168.203.129@tcp:38:kworker.0 00000800:00000100:3.0:1787828082.050926:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:3.0:1787828082.050940:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828082/real 1787828082] req@ffff8cb999f383c0 x1874672572507904/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828107 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828082.050954:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f383c0 x1874672572507904/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828107 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828082.050982:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828082.050986:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:3.0:1787828082.051014:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f383c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572507904:192.168.203.129@tcp:38:kworker.0 00000100:00100000:3.0:1787828082.051023:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f3a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572508032:192.168.203.129@tcp:38:kworker.0 00000800:00000100:3.0:1787828082.051051:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:3.0:1787828082.051065:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828082/real 1787828082] req@ffff8cb999f3a580 x1874672572508032/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828107 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828082.051078:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f3a580 x1874672572508032/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828107 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828082.051088:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828082.051091:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:3.0:1787828082.051095:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f3a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572508032:192.168.203.129@tcp:38:kworker.0 00010000:00010000:1.0:1787828082.882446:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787828083.906465:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787828084.930371:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787828085.954554:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787828086.978516:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787828087.170391:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828087.170407:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787828087.170414:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828087.170440:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 1039 00000100:00080000:1.0:1787828087.170458:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828087.170469:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:1.0:1787828087.170544:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f392c0 x1874672572508160/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828087.170591:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828087.170597:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787828087.170599:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828087.170604:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 1039 00000100:00080000:1.0:1787828087.170608:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828087.170644:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:1.0:1787828087.170660:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787828087.170681:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f3b0c0 x1874672572508288/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1787828087.170688:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f392c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572508160:192.168.203.129@tcp:250:kworker.0 00000100:00080000:1.0:1787828087.170726:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828087.170734:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1787828087.170736:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828087.170747:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 1039 00000100:00080000:1.0:1787828087.170753:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828087.170758:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:1.0:1787828087.170767:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787828087.170786:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f3b480 x1874672572508416/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1787828087.170842:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:3.0:1787828087.170928:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828087/real 1787828087] req@ffff8cb999f392c0 x1874672572508160/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828112 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828087.171000:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f392c0 x1874672572508160/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828112 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828087.171019:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828087.171025:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:3.0:1787828087.171078:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f392c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572508160:192.168.203.129@tcp:250:kworker.0 00000100:00100000:3.0:1787828087.171117:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f3b0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572508288:192.168.203.129@tcp:38:kworker.0 00000800:00000100:3.0:1787828087.171196:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:3.0:1787828087.171222:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828087/real 1787828087] req@ffff8cb999f3b0c0 x1874672572508288/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828112 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828087.171247:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f3b0c0 x1874672572508288/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828112 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828087.171316:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828087.171321:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:3.0:1787828087.171325:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f3b0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572508288:192.168.203.129@tcp:38:kworker.0 00000100:00100000:3.0:1787828087.171353:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f3b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572508416:192.168.203.129@tcp:38:kworker.0 00000800:00000100:3.0:1787828087.171389:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:3.0:1787828087.171423:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828087/real 1787828087] req@ffff8cb999f3b480 x1874672572508416/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828112 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828087.171466:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f3b480 x1874672572508416/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828112 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828087.171482:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828087.171488:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:3.0:1787828087.171495:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f3b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572508416:192.168.203.129@tcp:38:kworker.0 00010000:00010000:1.0:1787828088.002526:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787828089.026507:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787828090.050432:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787828091.074493:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787828092.098404:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787828092.290438:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828092.290454:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787828092.290461:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828092.290479:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 1044 00000100:00080000:2.0:1787828092.290497:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828092.290511:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:2.0:1787828092.290584:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8de00 x1874672572508544/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828092.290634:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828092.290639:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787828092.290642:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828092.290648:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 1044 00000100:00080000:2.0:1787828092.290655:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828092.290663:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:2.0:1787828092.290680:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787828092.290697:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8de00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572508544:192.168.203.129@tcp:250:kworker.0 00000100:00100000:2.0:1787828092.290703:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8c3c0 x1874672572508672/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828092.290764:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828092.290769:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1787828092.290772:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828092.290782:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 1044 00000800:00000100:3.0:1787828092.290794:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00080000:2.0:1787828092.290797:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828092.290804:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:2.0:1787828092.290812:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787828092.290832:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8d2c0 x1874672572508800/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1787828092.290879:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828092/real 1787828092] req@ffff8cb998a8de00 x1874672572508544/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828117 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828092.290897:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8de00 x1874672572508544/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828117 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828092.290909:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828092.290913:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:3.0:1787828092.290916:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8de00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572508544:192.168.203.129@tcp:250:kworker.0 00000100:00100000:3.0:1787828092.290955:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8c3c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572508672:192.168.203.129@tcp:38:kworker.0 00000800:00000100:3.0:1787828092.290980:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:3.0:1787828092.290993:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828092/real 1787828092] req@ffff8cb998a8c3c0 x1874672572508672/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828117 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828092.291008:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8c3c0 x1874672572508672/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828117 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828092.291017:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828092.291020:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:3.0:1787828092.291023:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8c3c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572508672:192.168.203.129@tcp:38:kworker.0 00000100:00100000:3.0:1787828092.291029:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8d2c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572508800:192.168.203.129@tcp:38:kworker.0 00000800:00000100:3.0:1787828092.291045:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:3.0:1787828092.291053:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828092/real 1787828092] req@ffff8cb998a8d2c0 x1874672572508800/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828117 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828092.291061:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8d2c0 x1874672572508800/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828117 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828092.291077:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828092.291080:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:3.0:1787828092.291082:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8d2c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572508800:192.168.203.129@tcp:38:kworker.0 00010000:00010000:1.0:1787828093.122515:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787828094.146587:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787828095.170527:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787828096.194510:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787828097.218540:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787828097.410370:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828097.410384:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787828097.410389:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828097.410402:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 1049 00000100:00080000:2.0:1787828097.410416:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828097.410428:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:2.0:1787828097.410474:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8fc00 x1874672572508928/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828097.410532:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828097.410540:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787828097.410545:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828097.410566:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 1049 00000100:00080000:2.0:1787828097.410574:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828097.410583:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:2.0:1787828097.410615:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787828097.410653:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8da40 x1874672572509056/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787828097.410665:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8fc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572508928:192.168.203.129@tcp:250:kworker.0 00000100:00080000:2.0:1787828097.410695:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828097.410705:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1787828097.410711:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828097.410724:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 1049 00000100:00080000:2.0:1787828097.410736:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828097.410748:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000800:00000100:0.0:1787828097.410762:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000008:00100000:2.0:1787828097.410763:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787828097.410797:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8e1c0 x1874672572509184/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787828097.410829:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828097/real 1787828097] req@ffff8cb998a8fc00 x1874672572508928/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828122 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828097.410860:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8fc00 x1874672572508928/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828122 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828097.410876:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828097.410908:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787828097.410922:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8fc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572508928:192.168.203.129@tcp:250:kworker.0 00000100:00100000:0.0:1787828097.410951:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8da40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572509056:192.168.203.129@tcp:38:kworker.0 00000800:00000100:0.0:1787828097.410988:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787828097.411005:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828097/real 1787828097] req@ffff8cb998a8da40 x1874672572509056/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828122 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828097.411022:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8da40 x1874672572509056/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828122 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828097.411055:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828097.411060:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787828097.411065:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8da40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572509056:192.168.203.129@tcp:38:kworker.0 00000100:00100000:0.0:1787828097.411073:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8e1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572509184:192.168.203.129@tcp:38:kworker.0 00000800:00000100:0.0:1787828097.411103:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787828097.411127:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828097/real 1787828097] req@ffff8cb998a8e1c0 x1874672572509184/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828122 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828097.411144:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8e1c0 x1874672572509184/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828122 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828097.411161:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828097.411170:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787828097.411177:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8e1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572509184:192.168.203.129@tcp:38:kworker.0 00010000:00010000:1.0:1787828098.242580:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787828099.266489:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787828100.290481:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787828101.314578:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787828102.338508:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787828102.530413:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828102.530431:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787828102.530438:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828102.530451:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 1054 00000100:00080000:0.0:1787828102.530465:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828102.530473:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:0.0:1787828102.530517:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c6a1c0 x1874672572509312/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828102.530559:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828102.530587:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787828102.530591:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828102.530597:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 1054 00000100:00080000:0.0:1787828102.530604:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828102.530611:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:0.0:1787828102.530625:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787828102.530641:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c6a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572509312:192.168.203.129@tcp:250:kworker.0 00000100:00100000:0.0:1787828102.530647:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c6a580 x1874672572509440/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828102.530665:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828102.530677:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1787828102.530680:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828102.530685:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 1054 00000100:00080000:0.0:1787828102.530689:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828102.530693:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:0.0:1787828102.530699:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787828102.530715:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c692c0 x1874672572509568/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1787828102.530775:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:3.0:1787828102.530841:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828102/real 1787828102] req@ffff8cb999c6a1c0 x1874672572509312/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828127 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828102.530894:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c6a1c0 x1874672572509312/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828127 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828102.530911:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828102.530917:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:3.0:1787828102.530923:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c6a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572509312:192.168.203.129@tcp:250:kworker.0 00000100:00100000:3.0:1787828102.530954:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c6a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572509440:192.168.203.129@tcp:38:kworker.0 00000800:00000100:3.0:1787828102.530989:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:3.0:1787828102.531006:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828102/real 1787828102] req@ffff8cb999c6a580 x1874672572509440/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828127 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828102.531023:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c6a580 x1874672572509440/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828127 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828102.531035:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828102.531039:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:3.0:1787828102.531044:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c6a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572509440:192.168.203.129@tcp:38:kworker.0 00000100:00100000:3.0:1787828102.531065:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c692c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572509568:192.168.203.129@tcp:38:kworker.0 00000800:00000100:3.0:1787828102.531094:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:3.0:1787828102.531108:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828102/real 1787828102] req@ffff8cb999c692c0 x1874672572509568/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828127 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828102.531123:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c692c0 x1874672572509568/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828127 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828102.531134:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828102.531138:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:3.0:1787828102.531143:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c692c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572509568:192.168.203.129@tcp:38:kworker.0 00010000:00010000:1.0:1787828103.362449:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787828104.386427:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787828105.410480:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787828106.434496:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787828107.458555:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787828107.650522:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828107.650535:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787828107.650541:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828107.650556:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 1059 00000100:00080000:2.0:1787828107.650571:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828107.650597:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:2.0:1787828107.650645:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8f0c0 x1874672572509696/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828107.650674:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828107.650677:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787828107.650679:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828107.650682:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 1059 00000100:00080000:2.0:1787828107.650721:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828107.650728:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:2.0:1787828107.650744:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787828107.650761:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8cf00 x1874672572509824/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828107.650777:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828107.650782:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1787828107.650785:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828107.650791:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 1059 00000100:00080000:2.0:1787828107.650796:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828107.650800:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:2.0:1787828107.650805:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787828107.650861:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8c780 x1874672572509952/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1787828107.650863:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8f0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572509696:192.168.203.129@tcp:250:kworker.0 00000800:00000100:3.0:1787828107.650982:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:3.0:1787828107.651057:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828107/real 1787828107] req@ffff8cb998a8f0c0 x1874672572509696/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828132 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828107.651072:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8f0c0 x1874672572509696/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828132 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828107.651085:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828107.651090:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:3.0:1787828107.651095:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8f0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572509696:192.168.203.129@tcp:250:kworker.0 00000100:00100000:3.0:1787828107.651126:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8cf00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572509824:192.168.203.129@tcp:38:kworker.0 00000800:00000100:3.0:1787828107.651181:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:3.0:1787828107.651207:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828107/real 1787828107] req@ffff8cb998a8cf00 x1874672572509824/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828132 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828107.651251:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8cf00 x1874672572509824/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828132 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828107.651257:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828107.651260:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:3.0:1787828107.651262:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8cf00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572509824:192.168.203.129@tcp:38:kworker.0 00000100:00100000:3.0:1787828107.651288:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8c780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572509952:192.168.203.129@tcp:38:kworker.0 00000800:00000100:3.0:1787828107.651326:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:3.0:1787828107.651338:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828107/real 1787828107] req@ffff8cb998a8c780 x1874672572509952/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828132 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828107.651350:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8c780 x1874672572509952/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828132 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828107.651359:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828107.651362:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:3.0:1787828107.651366:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8c780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572509952:192.168.203.129@tcp:38:kworker.0 00010000:00010000:1.0:1787828108.482453:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787828109.506472:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787828110.530414:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787828111.554481:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787828112.578500:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787828112.770434:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828112.770445:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787828112.770450:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828112.770461:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 1064 00000100:00080000:1.0:1787828112.770472:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828112.770480:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:1.0:1787828112.770520:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f38b40 x1874672572510080/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828112.770555:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828112.770559:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787828112.770561:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828112.770565:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 1064 00000100:00080000:1.0:1787828112.770568:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828112.770572:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:1.0:1787828112.770582:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787828112.770594:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f39e00 x1874672572510208/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828112.770605:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828112.770629:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00100000:3.0:1787828112.770631:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f38b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572510080:192.168.203.129@tcp:250:kworker.0 00000100:00080000:1.0:1787828112.770632:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828112.770644:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 1064 00000100:00080000:1.0:1787828112.770652:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828112.770656:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:1.0:1787828112.770662:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787828112.770676:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f38f00 x1874672572510336/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1787828112.770807:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f39e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572510208:192.168.203.129@tcp:38:kworker.0 00000100:00100000:3.0:1787828112.770909:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f38f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572510336:192.168.203.129@tcp:38:kworker.0 00010000:00010000:0.0:1787828113.602519:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787828114.626459:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787828115.650843:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:0.0:1787828115.842590:0:2384:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.129:988 00000400:00000100:0.0:1787828115.842644:0:2384:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.129@tcp at host 192.168.203.129 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:0.0:1787828115.842664:0:2384:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.129@tcp 00000800:00000100:0.0:1787828115.842729:0:2384:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.129@tcp 00000800:00000100:0.0:1787828115.842745:0:2384:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.129@tcp 00000100:00100000:3.0:1787828115.842786:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828112/real 1787828115] req@ffff8cb999f38f00 x1874672572510336/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828137 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828115.842829:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f38f00 x1874672572510336/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828137 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828115.842843:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828115.842862:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:3.0:1787828115.842868:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f38f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572510336:192.168.203.129@tcp:38:kworker.0 00000100:00100000:3.0:1787828115.842880:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828112/real 1787828115] req@ffff8cb999f38b40 x1874672572510080/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828137 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828115.842899:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f38b40 x1874672572510080/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828137 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828115.842909:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828115.842915:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:3.0:1787828115.842919:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f38b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572510080:192.168.203.129@tcp:250:kworker.0 00000100:00100000:3.0:1787828115.842929:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828112/real 1787828115] req@ffff8cb999f39e00 x1874672572510208/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828137 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828115.842947:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f39e00 x1874672572510208/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828137 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828115.842969:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828115.842974:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:3.0:1787828115.842979:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f39e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572510208:192.168.203.129@tcp:38:kworker.0 00010000:00010000:1.0:1787828116.675010:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787828117.699362:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787828117.890439:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828117.890454:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787828117.890460:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828117.890477:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 1070 00000100:00080000:2.0:1787828117.890494:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828117.890505:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:2.0:1787828117.890563:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8e580 x1874672572510464/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828117.890611:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828117.890638:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787828117.890641:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828117.890646:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 1070 00000100:00080000:2.0:1787828117.890653:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828117.890659:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:2.0:1787828117.890677:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787828117.890697:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8d680 x1874672572510592/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1787828117.890715:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8e580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572510464:192.168.203.129@tcp:250:kworker.0 00000100:00080000:2.0:1787828117.890719:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828117.890727:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1787828117.890731:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828117.890772:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 1070 00000100:00080000:2.0:1787828117.890781:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828117.890789:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:2.0:1787828117.890799:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787828117.890824:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8c000 x1874672572510720/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1787828117.890931:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:3.0:1787828117.891015:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828117/real 1787828117] req@ffff8cb998a8e580 x1874672572510464/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828142 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828117.891053:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8e580 x1874672572510464/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828142 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828117.891075:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828117.891083:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:3.0:1787828117.891091:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8e580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572510464:192.168.203.129@tcp:250:kworker.0 00000100:00100000:3.0:1787828117.891143:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8d680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572510592:192.168.203.129@tcp:38:kworker.0 00000800:00000100:3.0:1787828117.891194:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:3.0:1787828117.891222:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828117/real 1787828117] req@ffff8cb998a8d680 x1874672572510592/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828142 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828117.891246:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8d680 x1874672572510592/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828142 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828117.891318:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828117.891327:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:3.0:1787828117.891335:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8d680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572510592:192.168.203.129@tcp:38:kworker.0 00000100:00100000:3.0:1787828117.891350:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572510720:192.168.203.129@tcp:38:kworker.0 00000800:00000100:3.0:1787828117.891396:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:3.0:1787828117.891633:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828117/real 1787828117] req@ffff8cb998a8c000 x1874672572510720/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828142 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828117.891694:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8c000 x1874672572510720/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828142 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828117.891712:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828117.891720:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:3.0:1787828117.891726:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572510720:192.168.203.129@tcp:38:kworker.0 00010000:00010000:1.0:1787828118.722819:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787828119.747042:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787828120.771199:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787828121.794626:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787828122.818465:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787828123.010385:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828123.010394:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787828123.010397:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828123.010406:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 1075 00000100:00080000:1.0:1787828123.010415:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828123.010427:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:1.0:1787828123.010463:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f3b840 x1874672572510848/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828123.010485:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828123.010490:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787828123.010508:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828123.010513:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 1075 00000100:00080000:1.0:1787828123.010518:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828123.010522:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:3.0:1787828123.010530:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f3b840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572510848:192.168.203.129@tcp:250:kworker.0 00000008:00100000:1.0:1787828123.010532:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787828123.010546:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f39a40 x1874672572510976/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828123.010557:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828123.010560:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1787828123.010562:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828123.010566:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 1075 00000100:00080000:1.0:1787828123.010570:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828123.010573:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:1.0:1787828123.010577:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787828123.010587:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f39e00 x1874672572511104/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1787828123.010590:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:3.0:1787828123.010634:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828123/real 1787828123] req@ffff8cb999f3b840 x1874672572510848/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828148 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828123.010652:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f3b840 x1874672572510848/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828148 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828123.010659:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828123.010662:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:3.0:1787828123.010665:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f3b840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572510848:192.168.203.129@tcp:250:kworker.0 00000100:00100000:3.0:1787828123.010685:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f39a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572510976:192.168.203.129@tcp:38:kworker.0 00000800:00000100:3.0:1787828123.010721:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:3.0:1787828123.010734:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828123/real 1787828123] req@ffff8cb999f39a40 x1874672572510976/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828148 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828123.010743:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f39a40 x1874672572510976/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828148 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828123.010765:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828123.010768:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:3.0:1787828123.010773:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f39a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572510976:192.168.203.129@tcp:38:kworker.0 00000100:00100000:3.0:1787828123.010780:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f39e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572511104:192.168.203.129@tcp:38:kworker.0 00000800:00000100:3.0:1787828123.010807:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:3.0:1787828123.010820:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828123/real 1787828123] req@ffff8cb999f39e00 x1874672572511104/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828148 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828123.010831:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f39e00 x1874672572511104/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828148 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828123.010840:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828123.010843:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:3.0:1787828123.010847:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f39e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572511104:192.168.203.129@tcp:38:kworker.0 00010000:00010000:1.0:1787828123.842491:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787828124.866545:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787828125.890459:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787828126.914504:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787828127.938434:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787828128.130362:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828128.130374:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787828128.130380:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828128.130392:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 1080 00000100:00080000:1.0:1787828128.130404:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828128.130413:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:1.0:1787828128.130492:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f38b40 x1874672572511232/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828128.130539:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828128.130544:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787828128.130547:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828128.130552:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 1080 00000100:00080000:1.0:1787828128.130558:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828128.130563:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:1.0:1787828128.130577:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787828128.130593:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f38f00 x1874672572511360/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828128.130610:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1787828128.130613:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f38b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572511232:192.168.203.129@tcp:250:kworker.0 00000100:00080000:1.0:1787828128.130623:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1787828128.130630:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828128.130637:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 1080 00000100:00080000:1.0:1787828128.130641:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828128.130645:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:1.0:1787828128.130652:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787828128.130691:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f3b480 x1874672572511488/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1787828128.130745:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:3.0:1787828128.130796:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828128/real 1787828128] req@ffff8cb999f38b40 x1874672572511232/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828153 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828128.130833:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f38b40 x1874672572511232/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828153 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828128.130848:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828128.130853:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:3.0:1787828128.130870:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f38b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572511232:192.168.203.129@tcp:250:kworker.0 00000100:00100000:3.0:1787828128.130902:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f38f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572511360:192.168.203.129@tcp:38:kworker.0 00000800:00000100:3.0:1787828128.130941:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:3.0:1787828128.130958:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828128/real 1787828128] req@ffff8cb999f38f00 x1874672572511360/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828153 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828128.130974:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f38f00 x1874672572511360/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828153 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828128.130984:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828128.130989:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:3.0:1787828128.130993:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f38f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572511360:192.168.203.129@tcp:38:kworker.0 00000100:00100000:3.0:1787828128.131000:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f3b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572511488:192.168.203.129@tcp:38:kworker.0 00000800:00000100:3.0:1787828128.131019:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:3.0:1787828128.131029:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828128/real 1787828128] req@ffff8cb999f3b480 x1874672572511488/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828153 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828128.131043:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f3b480 x1874672572511488/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828153 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828128.131053:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828128.131057:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:3.0:1787828128.131060:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f3b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572511488:192.168.203.129@tcp:38:kworker.0 00010000:00010000:0.0:1787828128.962511:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787828129.986576:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787828131.010379:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787828132.034514:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787828133.058503:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787828133.250332:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828133.250340:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787828133.250344:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828133.250368:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 1085 00000100:00080000:1.0:1787828133.250378:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828133.250386:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:1.0:1787828133.250420:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f3b0c0 x1874672572511616/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828133.250449:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828133.250453:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787828133.250454:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828133.250458:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 1085 00000100:00080000:1.0:1787828133.250461:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828133.250464:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:1.0:1787828133.250473:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787828133.250486:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f3b0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572511616:192.168.203.129@tcp:250:kworker.0 00000100:00100000:1.0:1787828133.250487:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f392c0 x1874672572511744/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828133.250505:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828133.250508:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1787828133.250510:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828133.250514:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 1085 00000100:00080000:1.0:1787828133.250518:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828133.250521:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:1.0:1787828133.250525:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787828133.250537:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f3a580 x1874672572511872/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1787828133.250557:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787828133.250595:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828133/real 1787828133] req@ffff8cb999f3b0c0 x1874672572511616/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828158 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828133.250610:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f3b0c0 x1874672572511616/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828158 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828133.250654:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828133.250658:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787828133.250662:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f3b0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572511616:192.168.203.129@tcp:250:kworker.0 00000100:00100000:2.0:1787828133.250710:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f392c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572511744:192.168.203.129@tcp:38:kworker.0 00000800:00000100:2.0:1787828133.250747:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787828133.250765:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828133/real 1787828133] req@ffff8cb999f392c0 x1874672572511744/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828158 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828133.250781:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f392c0 x1874672572511744/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828158 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828133.250792:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828133.250814:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787828133.250819:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f392c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572511744:192.168.203.129@tcp:38:kworker.0 00000100:00100000:2.0:1787828133.250828:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f3a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572511872:192.168.203.129@tcp:38:kworker.0 00000800:00000100:2.0:1787828133.250851:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787828133.250864:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828133/real 1787828133] req@ffff8cb999f3a580 x1874672572511872/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828158 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828133.250877:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f3a580 x1874672572511872/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828158 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828133.250902:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828133.250905:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787828133.250908:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f3a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572511872:192.168.203.129@tcp:38:kworker.0 00010000:00010000:0.0:1787828134.082434:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787828135.106459:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787828136.130495:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787828137.154602:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787828138.178488:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787828138.370521:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828138.370537:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787828138.370544:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828138.370558:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 1090 00000100:00080000:2.0:1787828138.370571:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828138.370581:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:2.0:1787828138.370667:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8f480 x1874672572512000/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828138.370712:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828138.370718:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787828138.370721:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828138.370725:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 1090 00000100:00080000:2.0:1787828138.370739:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828138.370746:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:2.0:1787828138.370760:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787828138.370797:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8cb40 x1874672572512128/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1787828138.370816:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572512000:192.168.203.129@tcp:250:kworker.0 00000100:00080000:2.0:1787828138.370832:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828138.370843:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1787828138.370847:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828138.370859:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 1090 00000100:00080000:2.0:1787828138.370868:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828138.370878:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:2.0:1787828138.370909:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000800:00000100:3.0:1787828138.370949:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787828138.370953:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8e940 x1874672572512256/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1787828138.371079:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828138/real 1787828138] req@ffff8cb998a8f480 x1874672572512000/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828163 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828138.371104:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8f480 x1874672572512000/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828163 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828138.371118:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828138.371123:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:3.0:1787828138.371127:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572512000:192.168.203.129@tcp:250:kworker.0 00000100:00100000:3.0:1787828138.371180:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8cb40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572512128:192.168.203.129@tcp:38:kworker.0 00000800:00000100:3.0:1787828138.371222:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:3.0:1787828138.371235:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828138/real 1787828138] req@ffff8cb998a8cb40 x1874672572512128/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828163 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828138.371247:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8cb40 x1874672572512128/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828163 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828138.371256:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828138.371259:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:3.0:1787828138.371262:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8cb40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572512128:192.168.203.129@tcp:38:kworker.0 00000100:00100000:3.0:1787828138.371267:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8e940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572512256:192.168.203.129@tcp:38:kworker.0 00000800:00000100:3.0:1787828138.371284:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:3.0:1787828138.371292:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828138/real 1787828138] req@ffff8cb998a8e940 x1874672572512256/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828163 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828138.371320:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8e940 x1874672572512256/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828163 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828138.371343:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828138.371346:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:3.0:1787828138.371350:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8e940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572512256:192.168.203.129@tcp:38:kworker.0 00010000:00010000:0.0:1787828139.202553:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787828140.226411:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787828141.250465:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787828142.274469:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787828143.298546:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787828143.490485:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828143.490503:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787828143.490510:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828143.490527:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 1095 00000100:00080000:0.0:1787828143.490544:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828143.490556:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:0.0:1787828143.490653:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c68000 x1874672572512384/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828143.490698:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828143.490731:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787828143.490735:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828143.490742:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 1095 00000100:00080000:0.0:1787828143.490749:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828143.490755:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:3.0:1787828143.490770:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c68000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572512384:192.168.203.129@tcp:250:kworker.0 00000008:00100000:0.0:1787828143.490773:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787828143.490801:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c69680 x1874672572512512/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828143.490817:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828143.490822:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1787828143.490825:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828143.490831:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 1095 00000100:00080000:0.0:1787828143.490835:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828143.490840:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:0.0:1787828143.490846:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787828143.490861:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c6b480 x1874672572512640/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1787828143.490902:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:3.0:1787828143.490993:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828143/real 1787828143] req@ffff8cb999c68000 x1874672572512384/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828168 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828143.491024:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c68000 x1874672572512384/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828168 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828143.491040:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828143.491045:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:3.0:1787828143.491051:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c68000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572512384:192.168.203.129@tcp:250:kworker.0 00000100:00100000:3.0:1787828143.491090:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c69680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572512512:192.168.203.129@tcp:38:kworker.0 00000800:00000100:3.0:1787828143.491126:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:3.0:1787828143.491143:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828143/real 1787828143] req@ffff8cb999c69680 x1874672572512512/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828168 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828143.491158:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c69680 x1874672572512512/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828168 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828143.491169:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828143.491173:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:3.0:1787828143.491177:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c69680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572512512:192.168.203.129@tcp:38:kworker.0 00000100:00100000:3.0:1787828143.491198:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c6b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572512640:192.168.203.129@tcp:38:kworker.0 00000800:00000100:3.0:1787828143.491227:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:3.0:1787828143.491241:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828143/real 1787828143] req@ffff8cb999c6b480 x1874672572512640/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828168 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828143.491256:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c6b480 x1874672572512640/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828168 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828143.491307:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828143.491312:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:3.0:1787828143.491316:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c6b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572512640:192.168.203.129@tcp:38:kworker.0 00010000:00010000:0.0:1787828144.322453:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787828145.346421:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787828146.370531:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787828147.394524:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787828148.418612:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787828148.610570:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828148.610590:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787828148.610598:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828148.610618:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 1100 00000100:00080000:2.0:1787828148.610647:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828148.610663:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:2.0:1787828148.610727:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8f840 x1874672572512768/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828148.610766:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828148.610772:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787828148.610776:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828148.610781:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 1100 00000100:00080000:2.0:1787828148.610788:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828148.610794:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:2.0:1787828148.610807:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787828148.610824:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8f840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572512768:192.168.203.129@tcp:250:kworker.0 00000100:00100000:2.0:1787828148.610832:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8ed00 x1874672572512896/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828148.610866:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828148.610870:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1787828148.610875:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828148.610880:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 1100 00000100:00080000:2.0:1787828148.610901:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828148.610912:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:2.0:1787828148.610924:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787828148.610958:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8e940 x1874672572513024/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1787828148.610973:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:3.0:1787828148.611070:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828148/real 1787828148] req@ffff8cb998a8f840 x1874672572512768/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828173 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828148.611098:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8f840 x1874672572512768/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828173 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828148.611115:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828148.611121:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:3.0:1787828148.611126:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8f840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572512768:192.168.203.129@tcp:250:kworker.0 00000100:00100000:3.0:1787828148.611165:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8ed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572512896:192.168.203.129@tcp:38:kworker.0 00000800:00000100:3.0:1787828148.611194:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:3.0:1787828148.611209:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828148/real 1787828148] req@ffff8cb998a8ed00 x1874672572512896/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828173 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828148.611248:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8ed00 x1874672572512896/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828173 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828148.611303:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828148.611311:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:3.0:1787828148.611318:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8ed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572512896:192.168.203.129@tcp:38:kworker.0 00000100:00100000:3.0:1787828148.611334:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8e940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572513024:192.168.203.129@tcp:38:kworker.0 00000800:00000100:3.0:1787828148.611384:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:3.0:1787828148.611409:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828148/real 1787828148] req@ffff8cb998a8e940 x1874672572513024/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828173 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828148.611427:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8e940 x1874672572513024/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828173 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828148.611438:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828148.611443:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:3.0:1787828148.611447:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8e940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572513024:192.168.203.129@tcp:38:kworker.0 00010000:00010000:0.0:1787828149.442485:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787828150.466560:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787828151.490464:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787828152.514476:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787828153.538539:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787828153.730353:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828153.730364:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787828153.730369:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828153.730380:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 1105 00000100:00080000:2.0:1787828153.730392:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828153.730399:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:2.0:1787828153.730441:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8cb40 x1874672572513152/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828153.730468:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828153.730472:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787828153.730473:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828153.730476:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 1105 00000100:00080000:2.0:1787828153.730479:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828153.730481:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:2.0:1787828153.730502:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787828153.730514:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8f480 x1874672572513280/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828153.730523:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1787828153.730533:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8cb40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572513152:192.168.203.129@tcp:250:kworker.0 00000100:00080000:2.0:1787828153.730544:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1787828153.730546:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828153.730550:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 1105 00000100:00080000:2.0:1787828153.730554:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828153.730558:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:2.0:1787828153.730562:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787828153.730572:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8c000 x1874672572513408/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1787828153.730623:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:3.0:1787828153.730673:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828153/real 1787828153] req@ffff8cb998a8cb40 x1874672572513152/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828178 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828153.730691:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8cb40 x1874672572513152/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828178 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828153.730720:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828153.730724:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:3.0:1787828153.730729:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8cb40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572513152:192.168.203.129@tcp:250:kworker.0 00000100:00100000:3.0:1787828153.730756:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572513280:192.168.203.129@tcp:38:kworker.0 00000800:00000100:3.0:1787828153.730778:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:3.0:1787828153.730790:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828153/real 1787828153] req@ffff8cb998a8f480 x1874672572513280/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828178 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828153.730802:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8f480 x1874672572513280/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828178 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828153.730810:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828153.730813:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:3.0:1787828153.730816:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572513280:192.168.203.129@tcp:38:kworker.0 00000100:00100000:3.0:1787828153.730821:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572513408:192.168.203.129@tcp:38:kworker.0 00000800:00000100:3.0:1787828153.730837:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:3.0:1787828153.730845:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828153/real 1787828153] req@ffff8cb998a8c000 x1874672572513408/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828178 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828153.730856:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8c000 x1874672572513408/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828178 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828153.730863:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828153.730866:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:3.0:1787828153.730869:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572513408:192.168.203.129@tcp:38:kworker.0 00010000:00010000:0.0:1787828154.562500:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787828155.586642:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787828156.610552:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787828157.634532:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787828158.658492:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787828158.850364:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828158.850376:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787828158.850382:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828158.850395:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 1111 00000100:00080000:2.0:1787828158.850409:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828158.850418:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:2.0:1787828158.850494:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8d680 x1874672572513536/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828158.850553:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828158.850560:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787828158.850563:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828158.850569:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 1111 00000100:00080000:2.0:1787828158.850575:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828158.850583:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:2.0:1787828158.850599:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787828158.850622:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8e580 x1874672572513664/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1787828158.850637:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8d680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572513536:192.168.203.129@tcp:250:kworker.0 00000100:00080000:2.0:1787828158.850659:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828158.850663:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1787828158.850665:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828158.850671:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 1111 00000100:00080000:2.0:1787828158.850676:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828158.850681:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:2.0:1787828158.850688:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787828158.850702:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8c780 x1874672572513792/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1787828158.850732:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:3.0:1787828158.850789:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828158/real 1787828158] req@ffff8cb998a8d680 x1874672572513536/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828183 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828158.850809:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8d680 x1874672572513536/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828183 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828158.850822:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828158.850826:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:3.0:1787828158.850831:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8d680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572513536:192.168.203.129@tcp:250:kworker.0 00000100:00100000:3.0:1787828158.850860:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8e580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572513664:192.168.203.129@tcp:38:kworker.0 00000800:00000100:3.0:1787828158.850881:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:3.0:1787828158.850893:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828158/real 1787828158] req@ffff8cb998a8e580 x1874672572513664/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828183 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828158.850904:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8e580 x1874672572513664/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828183 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828158.850912:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828158.850915:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:3.0:1787828158.850918:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8e580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572513664:192.168.203.129@tcp:38:kworker.0 00000100:00100000:3.0:1787828158.850924:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8c780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572513792:192.168.203.129@tcp:38:kworker.0 00000800:00000100:3.0:1787828158.850938:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:3.0:1787828158.850947:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828158/real 1787828158] req@ffff8cb998a8c780 x1874672572513792/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828183 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828158.850973:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8c780 x1874672572513792/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828183 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828158.850981:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828158.850984:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:3.0:1787828158.850987:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8c780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572513792:192.168.203.129@tcp:38:kworker.0 00010000:00010000:0.0:1787828159.682544:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787828160.706529:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787828161.730390:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787828162.754493:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787828163.778520:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787828163.970555:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828163.970568:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787828163.970574:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828163.970588:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 1116 00000100:00080000:1.0:1787828163.970602:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828163.970613:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:1.0:1787828163.970675:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f383c0 x1874672572513920/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828163.970708:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828163.970714:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787828163.970716:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828163.970721:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 1116 00000100:00080000:1.0:1787828163.970726:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828163.970730:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:1.0:1787828163.970742:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787828163.970756:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f3a1c0 x1874672572514048/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828163.970769:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828163.970774:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1787828163.970777:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828163.970784:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 1116 00000100:00080000:1.0:1787828163.970787:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828163.970791:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:1.0:1787828163.970819:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787828163.970829:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f383c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572513920:192.168.203.129@tcp:250:kworker.0 00000100:00100000:1.0:1787828163.970852:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f3a580 x1874672572514176/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1787828163.970936:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:3.0:1787828163.971006:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828163/real 1787828163] req@ffff8cb999f383c0 x1874672572513920/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828188 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828163.971033:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f383c0 x1874672572513920/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828188 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828163.971048:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828163.971053:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:3.0:1787828163.971059:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f383c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572513920:192.168.203.129@tcp:250:kworker.0 00000100:00100000:3.0:1787828163.971069:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f3a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572514048:192.168.203.129@tcp:38:kworker.0 00000800:00000100:3.0:1787828163.971102:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:3.0:1787828163.971118:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828163/real 1787828163] req@ffff8cb999f3a1c0 x1874672572514048/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828188 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828163.971133:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f3a1c0 x1874672572514048/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828188 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828163.971165:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828163.971170:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:3.0:1787828163.971174:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f3a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572514048:192.168.203.129@tcp:38:kworker.0 00000100:00100000:3.0:1787828163.971208:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f3a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572514176:192.168.203.129@tcp:38:kworker.0 00000800:00000100:3.0:1787828163.971236:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:3.0:1787828163.971250:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828163/real 1787828163] req@ffff8cb999f3a580 x1874672572514176/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828188 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828163.971308:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f3a580 x1874672572514176/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828188 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828163.971320:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828163.971324:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:3.0:1787828163.971328:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f3a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572514176:192.168.203.129@tcp:38:kworker.0 00010000:00010000:0.0:1787828164.802511:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787828165.826561:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787828166.850450:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787828167.874506:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787828168.898501:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787828169.090352:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828169.090360:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787828169.090364:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828169.090372:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 1121 00000100:00080000:2.0:1787828169.090382:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828169.090400:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:2.0:1787828169.090432:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8cf00 x1874672572514304/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828169.090455:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828169.090458:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787828169.090460:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828169.090463:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 1121 00000100:00080000:2.0:1787828169.090466:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828169.090468:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:2.0:1787828169.090489:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787828169.090499:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8f0c0 x1874672572514432/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828169.090509:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828169.090511:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1787828169.090513:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828169.090516:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 1121 00000100:00080000:2.0:1787828169.090522:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828169.090529:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:3.0:1787828169.090530:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8cf00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572514304:192.168.203.129@tcp:250:kworker.0 00000008:00100000:2.0:1787828169.090534:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787828169.090555:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8e1c0 x1874672572514560/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1787828169.090649:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:3.0:1787828169.090719:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828169/real 1787828169] req@ffff8cb998a8cf00 x1874672572514304/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828194 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828169.090747:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8cf00 x1874672572514304/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828194 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828169.090764:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828169.090771:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:3.0:1787828169.090792:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8cf00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572514304:192.168.203.129@tcp:250:kworker.0 00000100:00100000:3.0:1787828169.090855:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8f0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572514432:192.168.203.129@tcp:38:kworker.0 00000800:00000100:3.0:1787828169.090999:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:3.0:1787828169.091021:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828169/real 1787828169] req@ffff8cb998a8f0c0 x1874672572514432/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828194 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828169.091039:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8f0c0 x1874672572514432/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828194 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828169.091050:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828169.091314:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:3.0:1787828169.091324:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8f0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572514432:192.168.203.129@tcp:38:kworker.0 00000100:00100000:3.0:1787828169.091418:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8e1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572514560:192.168.203.129@tcp:38:kworker.0 00000800:00000100:3.0:1787828169.091454:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:3.0:1787828169.091473:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828169/real 1787828169] req@ffff8cb998a8e1c0 x1874672572514560/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828194 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828169.091488:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8e1c0 x1874672572514560/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828194 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828169.091499:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828169.091504:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:3.0:1787828169.091508:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8e1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572514560:192.168.203.129@tcp:38:kworker.0 00010000:00010000:0.0:1787828169.922606:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787828170.946524:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787828171.970583:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787828172.994502:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787828174.018494:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787828174.210381:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828174.210425:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787828174.210432:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828174.210448:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 1126 00000100:00080000:2.0:1787828174.210466:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828174.210477:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:2.0:1787828174.210545:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8da40 x1874672572514688/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828174.210590:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828174.210595:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787828174.210598:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828174.210603:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 1126 00000100:00080000:2.0:1787828174.210640:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828174.210651:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:3.0:1787828174.210668:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8da40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572514688:192.168.203.129@tcp:250:kworker.0 00000008:00100000:2.0:1787828174.210672:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787828174.210710:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8fc00 x1874672572514816/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828174.210735:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828174.210740:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1787828174.210743:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828174.210749:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 1126 00000100:00080000:2.0:1787828174.210776:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828174.210782:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:2.0:1787828174.210790:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000800:00000100:3.0:1787828174.210817:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787828174.210835:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8d2c0 x1874672572514944/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1787828174.210896:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828174/real 1787828174] req@ffff8cb998a8da40 x1874672572514688/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828199 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828174.210918:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8da40 x1874672572514688/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828199 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828174.210940:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828174.210944:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:3.0:1787828174.210948:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8da40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572514688:192.168.203.129@tcp:250:kworker.0 00000100:00100000:3.0:1787828174.210978:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8fc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572514816:192.168.203.129@tcp:38:kworker.0 00000800:00000100:3.0:1787828174.211001:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:3.0:1787828174.211013:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828174/real 1787828174] req@ffff8cb998a8fc00 x1874672572514816/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828199 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828174.211025:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8fc00 x1874672572514816/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828199 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828174.211033:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828174.211036:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:3.0:1787828174.211039:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8fc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572514816:192.168.203.129@tcp:38:kworker.0 00000100:00100000:3.0:1787828174.211044:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8d2c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572514944:192.168.203.129@tcp:38:kworker.0 00000800:00000100:3.0:1787828174.211065:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:3.0:1787828174.211081:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828174/real 1787828174] req@ffff8cb998a8d2c0 x1874672572514944/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828199 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828174.211099:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8d2c0 x1874672572514944/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828199 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828174.211130:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828174.211135:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:3.0:1787828174.211140:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8d2c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572514944:192.168.203.129@tcp:38:kworker.0 00010000:00010000:0.0:1787828175.042537:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787828176.066561:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787828177.090514:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787828178.114507:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787828179.138559:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787828179.330453:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828179.330464:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787828179.330469:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828179.330480:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 1131 00000100:00080000:0.0:1787828179.330492:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828179.330520:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:0.0:1787828179.330556:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c6bc00 x1874672572515072/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828179.330590:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828179.330594:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787828179.330596:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828179.330600:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 1131 00000100:00080000:0.0:1787828179.330605:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828179.330608:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:0.0:1787828179.330618:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787828179.330634:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c68f00 x1874672572515200/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1787828179.330651:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c6bc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572515072:192.168.203.129@tcp:250:kworker.0 00000100:00080000:0.0:1787828179.330656:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828179.330665:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1787828179.330668:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828179.330676:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 1131 00000100:00080000:0.0:1787828179.330683:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828179.330689:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:0.0:1787828179.330699:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787828179.330720:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c69e00 x1874672572515328/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1787828179.330799:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c68f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572515200:192.168.203.129@tcp:38:kworker.0 00000100:00100000:3.0:1787828179.330856:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c69e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572515328:192.168.203.129@tcp:38:kworker.0 00010000:00010000:0.0:1787828180.162416:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787828181.186513:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787828182.210685:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:1.0:1787828182.402730:0:2381:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.129:988 00000400:00000100:1.0:1787828182.402798:0:2381:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.129@tcp at host 192.168.203.129 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:1.0:1787828182.402819:0:2381:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.129@tcp 00000800:00000100:1.0:1787828182.402871:0:2381:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.129@tcp 00000800:00000100:1.0:1787828182.402884:0:2381:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.129@tcp 00000100:00100000:3.0:1787828182.402924:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828179/real 1787828182] req@ffff8cb999c68f00 x1874672572515200/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828204 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828182.402954:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c68f00 x1874672572515200/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828204 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828182.402970:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828182.402976:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:3.0:1787828182.402982:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c68f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572515200:192.168.203.129@tcp:38:kworker.0 00000100:00100000:3.0:1787828182.402992:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828179/real 1787828182] req@ffff8cb999c69e00 x1874672572515328/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828204 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828182.403006:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c69e00 x1874672572515328/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828204 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828182.403014:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828182.403018:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:3.0:1787828182.403054:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c69e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572515328:192.168.203.129@tcp:38:kworker.0 00000100:00100000:3.0:1787828182.403063:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828179/real 1787828182] req@ffff8cb999c6bc00 x1874672572515072/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828204 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828182.403079:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c6bc00 x1874672572515072/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828204 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828182.403088:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828182.403092:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:3.0:1787828182.403095:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c6bc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572515072:192.168.203.129@tcp:250:kworker.0 00010000:00010000:1.0:1787828183.234940:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787828184.258518:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787828184.450440:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828184.450452:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787828184.450458:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828184.450470:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 1136 00000100:00080000:3.0:1787828184.450482:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828184.450490:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:3.0:1787828184.450528:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb992316580 x1874672572515456/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828184.450563:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828184.450567:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787828184.450570:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828184.450576:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 1136 00000100:00080000:3.0:1787828184.450582:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828184.450588:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:3.0:1787828184.450603:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787828184.450655:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb992316580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572515456:192.168.203.129@tcp:250:kworker.0 00000100:00100000:3.0:1787828184.450674:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb992314780 x1874672572515584/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828184.450713:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828184.450721:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1787828184.450725:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828184.450734:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 1136 00000100:00080000:3.0:1787828184.450744:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000800:00000100:2.0:1787828184.450757:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00080000:3.0:1787828184.450759:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:3.0:1787828184.450794:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787828184.450820:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb992314b40 x1874672572515712/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787828184.450867:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828184/real 1787828184] req@ffff8cb992316580 x1874672572515456/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828209 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828184.450902:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb992316580 x1874672572515456/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828209 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828184.450944:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828184.450952:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787828184.450959:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb992316580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572515456:192.168.203.129@tcp:250:kworker.0 00000100:00100000:2.0:1787828184.450996:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb992314780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572515584:192.168.203.129@tcp:38:kworker.0 00000800:00000100:2.0:1787828184.451041:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787828184.451066:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828184/real 1787828184] req@ffff8cb992314780 x1874672572515584/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828209 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828184.451089:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb992314780 x1874672572515584/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828209 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828184.451105:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828184.451111:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787828184.451117:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb992314780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572515584:192.168.203.129@tcp:38:kworker.0 00000100:00100000:2.0:1787828184.451129:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb992314b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572515712:192.168.203.129@tcp:38:kworker.0 00000800:00000100:2.0:1787828184.451165:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787828184.451188:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828184/real 1787828184] req@ffff8cb992314b40 x1874672572515712/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828209 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828184.451210:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb992314b40 x1874672572515712/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828209 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828184.451226:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828184.451230:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787828184.451234:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb992314b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572515712:192.168.203.129@tcp:38:kworker.0 00010000:00010000:1.0:1787828185.282503:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787828186.306644:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787828187.330639:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787828188.354935:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787828189.378533:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787828189.570373:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828189.570393:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787828189.570399:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828189.570413:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 1141 00000100:00080000:3.0:1787828189.570427:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828189.570439:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:3.0:1787828189.570490:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb992317480 x1874672572515840/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828189.570534:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828189.570539:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787828189.570541:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828189.570545:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 1141 00000100:00080000:3.0:1787828189.570550:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828189.570576:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:3.0:1787828189.570588:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787828189.570590:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb992317480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572515840:192.168.203.129@tcp:250:kworker.0 00000100:00100000:3.0:1787828189.570606:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb992314f00 x1874672572515968/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828189.570621:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828189.570625:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1787828189.570628:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828189.570632:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 1141 00000100:00080000:3.0:1787828189.570649:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828189.570654:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:3.0:1787828189.570660:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000800:00000100:2.0:1787828189.570666:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:3.0:1787828189.570677:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb992315e00 x1874672572516096/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787828189.570719:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828189/real 1787828189] req@ffff8cb992317480 x1874672572515840/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828214 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828189.570740:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb992317480 x1874672572515840/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828214 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828189.570753:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828189.570757:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787828189.570761:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb992317480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572515840:192.168.203.129@tcp:250:kworker.0 00000100:00100000:2.0:1787828189.570788:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb992314f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572515968:192.168.203.129@tcp:38:kworker.0 00000800:00000100:2.0:1787828189.570810:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787828189.570822:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828189/real 1787828189] req@ffff8cb992314f00 x1874672572515968/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828214 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828189.570835:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb992314f00 x1874672572515968/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828214 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828189.570843:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828189.570847:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787828189.570851:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb992314f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572515968:192.168.203.129@tcp:38:kworker.0 00000100:00100000:2.0:1787828189.570856:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb992315e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572516096:192.168.203.129@tcp:38:kworker.0 00000800:00000100:2.0:1787828189.570872:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787828189.570905:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828189/real 1787828189] req@ffff8cb992315e00 x1874672572516096/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828214 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828189.570935:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb992315e00 x1874672572516096/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828214 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828189.570949:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828189.570955:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787828189.570961:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb992315e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572516096:192.168.203.129@tcp:38:kworker.0 00010000:00010000:2.0:1787828190.402468:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787828191.426441:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787828192.450420:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787828193.474390:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787828194.498574:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787828194.690326:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828194.690336:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787828194.690340:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828194.690350:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 1146 00000100:00080000:3.0:1787828194.690369:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828194.690377:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:3.0:1787828194.690413:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb992316940 x1874672572516224/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828194.690438:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828194.690442:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787828194.690458:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828194.690463:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 1146 00000100:00080000:3.0:1787828194.690468:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828194.690472:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:3.0:1787828194.690484:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787828194.690495:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb992316940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572516224:192.168.203.129@tcp:250:kworker.0 00000100:00100000:3.0:1787828194.690499:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb9923161c0 x1874672572516352/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828194.690514:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828194.690517:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1787828194.690519:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828194.690524:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 1146 00000100:00080000:3.0:1787828194.690527:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828194.690529:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:3.0:1787828194.690535:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787828194.690545:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb992315a40 x1874672572516480/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1787828194.690578:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787828194.690618:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828194/real 1787828194] req@ffff8cb992316940 x1874672572516224/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828219 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828194.690635:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb992316940 x1874672572516224/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828219 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828194.690645:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828194.690649:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787828194.690653:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb992316940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572516224:192.168.203.129@tcp:250:kworker.0 00000100:00100000:2.0:1787828194.690684:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb9923161c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572516352:192.168.203.129@tcp:38:kworker.0 00000800:00000100:2.0:1787828194.690705:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787828194.690716:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828194/real 1787828194] req@ffff8cb9923161c0 x1874672572516352/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828219 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828194.690744:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb9923161c0 x1874672572516352/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828219 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828194.690753:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828194.690755:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787828194.690759:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb9923161c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572516352:192.168.203.129@tcp:38:kworker.0 00000100:00100000:2.0:1787828194.690765:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb992315a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572516480:192.168.203.129@tcp:38:kworker.0 00000800:00000100:2.0:1787828194.690783:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787828194.690793:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828194/real 1787828194] req@ffff8cb992315a40 x1874672572516480/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828219 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828194.690804:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb992315a40 x1874672572516480/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828219 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828194.690811:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828194.690813:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787828194.690816:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb992315a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572516480:192.168.203.129@tcp:38:kworker.0 00010000:00010000:0.0:1787828195.522393:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787828196.546388:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787828197.570435:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787828198.594369:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787828199.618374:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787828199.810345:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828199.810352:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787828199.810356:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828199.810364:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 1151 00000100:00080000:0.0:1787828199.810373:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828199.810379:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:0.0:1787828199.810410:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c69a40 x1874672572516608/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828199.810431:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828199.810434:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787828199.810435:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828199.810438:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 1151 00000100:00080000:0.0:1787828199.810440:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828199.810443:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:0.0:1787828199.810450:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787828199.810459:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c6b840 x1874672572516736/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828199.810467:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828199.810483:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00100000:2.0:1787828199.810483:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c69a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572516608:192.168.203.129@tcp:250:kworker.0 00000100:00080000:0.0:1787828199.810486:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828199.810494:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 1151 00000100:00080000:0.0:1787828199.810497:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828199.810500:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:0.0:1787828199.810503:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787828199.810513:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c6bc00 x1874672572516864/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1787828199.810550:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787828199.810591:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828199/real 1787828199] req@ffff8cb999c69a40 x1874672572516608/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828224 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828199.810609:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c69a40 x1874672572516608/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828224 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828199.810627:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828199.810631:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787828199.810636:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c69a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572516608:192.168.203.129@tcp:250:kworker.0 00000100:00100000:2.0:1787828199.810643:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c6b840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572516736:192.168.203.129@tcp:38:kworker.0 00000800:00000100:2.0:1787828199.810668:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787828199.810681:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828199/real 1787828199] req@ffff8cb999c6b840 x1874672572516736/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828224 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828199.810697:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c6b840 x1874672572516736/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828224 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828199.810705:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828199.810709:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787828199.810712:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c6b840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572516736:192.168.203.129@tcp:38:kworker.0 00000100:00100000:2.0:1787828199.810736:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c6bc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572516864:192.168.203.129@tcp:38:kworker.0 00000800:00000100:2.0:1787828199.810755:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787828199.810766:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828199/real 1787828199] req@ffff8cb999c6bc00 x1874672572516864/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828224 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828199.810778:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c6bc00 x1874672572516864/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828224 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828199.810786:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828199.810790:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787828199.810793:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c6bc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572516864:192.168.203.129@tcp:38:kworker.0 00010000:00010000:0.0:1787828200.642465:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787828201.666442:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787828202.690393:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787828203.714407:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787828204.738398:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787828204.930372:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828204.930378:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787828204.930381:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828204.930390:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 1157 00000100:00080000:2.0:1787828204.930399:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828204.930406:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:2.0:1787828204.930434:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8c3c0 x1874672572516992/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828204.930455:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828204.930458:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787828204.930460:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828204.930463:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 1157 00000100:00080000:2.0:1787828204.930466:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828204.930469:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:2.0:1787828204.930478:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787828204.930488:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8de00 x1874672572517120/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787828204.930497:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8c3c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572516992:192.168.203.129@tcp:250:kworker.0 00000100:00080000:2.0:1787828204.930498:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828204.930505:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1787828204.930508:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828204.930513:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 1157 00000100:00080000:2.0:1787828204.930515:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828204.930518:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:2.0:1787828204.930529:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787828204.930541:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8d2c0 x1874672572517248/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1787828204.930569:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787828204.930613:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828204/real 1787828204] req@ffff8cb998a8c3c0 x1874672572516992/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828229 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828204.930635:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8c3c0 x1874672572516992/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828229 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828204.930648:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828204.930654:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787828204.930659:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8c3c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572516992:192.168.203.129@tcp:250:kworker.0 00000100:00100000:1.0:1787828204.930683:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8de00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572517120:192.168.203.129@tcp:38:kworker.0 00000800:00000100:1.0:1787828204.930709:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787828204.930723:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828204/real 1787828204] req@ffff8cb998a8de00 x1874672572517120/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828229 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828204.930754:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8de00 x1874672572517120/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828229 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828204.930765:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828204.930770:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787828204.930774:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8de00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572517120:192.168.203.129@tcp:38:kworker.0 00000100:00100000:1.0:1787828204.930782:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8d2c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572517248:192.168.203.129@tcp:38:kworker.0 00000800:00000100:1.0:1787828204.930811:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787828204.930827:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828204/real 1787828204] req@ffff8cb998a8d2c0 x1874672572517248/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828229 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828204.930845:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8d2c0 x1874672572517248/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828229 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828204.930857:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828204.930861:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787828204.930865:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8d2c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572517248:192.168.203.129@tcp:38:kworker.0 00010000:00010000:2.0:1787828205.762442:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787828206.786412:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787828207.810435:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787828208.834476:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787828209.858395:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787828210.050388:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828210.050398:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787828210.050402:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828210.050412:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 1162 00000100:00080000:0.0:1787828210.050422:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828210.050430:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:0.0:1787828210.050461:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c69e00 x1874672572517376/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828210.050486:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828210.050489:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787828210.050510:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828210.050514:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 1162 00000100:00080000:0.0:1787828210.050518:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828210.050521:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:0.0:1787828210.050530:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787828210.050543:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c68f00 x1874672572517504/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787828210.050543:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c69e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572517376:192.168.203.129@tcp:250:kworker.0 00000100:00080000:0.0:1787828210.050558:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828210.050561:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1787828210.050562:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828210.050565:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 1162 00000100:00080000:0.0:1787828210.050568:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828210.050570:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:0.0:1787828210.050573:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787828210.050582:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c6b480 x1874672572517632/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1787828210.050611:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787828210.050654:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828210/real 1787828210] req@ffff8cb999c69e00 x1874672572517376/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828235 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828210.050672:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c69e00 x1874672572517376/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828235 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828210.050698:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828210.050702:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787828210.050706:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c69e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572517376:192.168.203.129@tcp:250:kworker.0 00000100:00100000:1.0:1787828210.050730:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c68f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572517504:192.168.203.129@tcp:38:kworker.0 00000800:00000100:1.0:1787828210.050754:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787828210.050768:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828210/real 1787828210] req@ffff8cb999c68f00 x1874672572517504/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828235 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828210.050782:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c68f00 x1874672572517504/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828235 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828210.050790:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828210.050793:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787828210.050797:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c68f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572517504:192.168.203.129@tcp:38:kworker.0 00000100:00100000:1.0:1787828210.050803:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c6b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572517632:192.168.203.129@tcp:38:kworker.0 00000800:00000100:1.0:1787828210.050822:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787828210.050834:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828210/real 1787828210] req@ffff8cb999c6b480 x1874672572517632/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828235 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828210.050847:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c6b480 x1874672572517632/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828235 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828210.050856:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828210.050860:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787828210.050863:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c6b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572517632:192.168.203.129@tcp:38:kworker.0 00010000:00010000:0.0:1787828210.882451:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787828211.906422:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787828212.930417:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787828213.954442:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787828214.978477:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787828215.170423:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828215.170433:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787828215.170437:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828215.170448:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 1167 00000100:00080000:3.0:1787828215.170461:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828215.170467:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:3.0:1787828215.170511:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb9923170c0 x1874672572517760/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828215.170562:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828215.170565:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787828215.170567:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828215.170570:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 1167 00000100:00080000:3.0:1787828215.170573:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828215.170576:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:3.0:1787828215.170585:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787828215.170596:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb992315a40 x1874672572517888/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828215.170603:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828215.170606:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1787828215.170607:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828215.170609:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 1167 00000100:00080000:3.0:1787828215.170611:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828215.170613:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:3.0:1787828215.170616:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787828215.170623:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb9923161c0 x1874672572518016/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787828215.170666:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb9923170c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572517760:192.168.203.129@tcp:250:kworker.0 00000800:00000100:1.0:1787828215.170737:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787828215.170783:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828215/real 1787828215] req@ffff8cb9923170c0 x1874672572517760/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828240 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828215.170799:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb9923170c0 x1874672572517760/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828240 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828215.170808:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828215.170811:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787828215.170814:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb9923170c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572517760:192.168.203.129@tcp:250:kworker.0 00000100:00100000:1.0:1787828215.170819:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb992315a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572517888:192.168.203.129@tcp:38:kworker.0 00000800:00000100:1.0:1787828215.170837:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787828215.170845:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828215/real 1787828215] req@ffff8cb992315a40 x1874672572517888/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828240 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828215.170853:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb992315a40 x1874672572517888/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828240 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828215.170859:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828215.170861:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787828215.170863:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb992315a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572517888:192.168.203.129@tcp:38:kworker.0 00000100:00100000:1.0:1787828215.170867:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb9923161c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572518016:192.168.203.129@tcp:38:kworker.0 00000800:00000100:1.0:1787828215.170877:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787828215.170882:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828215/real 1787828215] req@ffff8cb9923161c0 x1874672572518016/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828240 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828215.170903:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb9923161c0 x1874672572518016/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828240 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828215.170909:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828215.170911:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787828215.170913:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb9923161c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572518016:192.168.203.129@tcp:38:kworker.0 00010000:00010000:0.0:1787828216.002423:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787828217.026436:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787828218.050418:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787828219.074401:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787828220.098415:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787828220.290372:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828220.290381:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787828220.290401:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828220.290409:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 1172 00000100:00080000:0.0:1787828220.290418:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828220.290424:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:0.0:1787828220.290459:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c69680 x1874672572518144/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828220.290491:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828220.290494:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787828220.290495:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828220.290498:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 1172 00000100:00080000:0.0:1787828220.290501:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828220.290505:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:0.0:1787828220.290513:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787828220.290523:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c68000 x1874672572518272/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828220.290532:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828220.290535:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1787828220.290536:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828220.290542:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 1172 00000100:00100000:1.0:1787828220.290542:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c69680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572518144:192.168.203.129@tcp:250:kworker.0 00000100:00080000:0.0:1787828220.290547:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828220.290552:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:0.0:1787828220.290555:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787828220.290562:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c692c0 x1874672572518400/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1787828220.290600:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787828220.290644:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828220/real 1787828220] req@ffff8cb999c69680 x1874672572518144/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828245 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828220.290661:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c69680 x1874672572518144/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828245 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828220.290670:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828220.290673:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787828220.290676:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c69680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572518144:192.168.203.129@tcp:250:kworker.0 00000100:00100000:1.0:1787828220.290697:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c68000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572518272:192.168.203.129@tcp:38:kworker.0 00000800:00000100:1.0:1787828220.290714:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787828220.290723:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828220/real 1787828220] req@ffff8cb999c68000 x1874672572518272/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828245 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828220.290731:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c68000 x1874672572518272/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828245 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828220.290746:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828220.290749:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787828220.290751:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c68000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572518272:192.168.203.129@tcp:38:kworker.0 00000100:00100000:1.0:1787828220.290755:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c692c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572518400:192.168.203.129@tcp:38:kworker.0 00000800:00000100:1.0:1787828220.290769:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787828220.290776:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828220/real 1787828220] req@ffff8cb999c692c0 x1874672572518400/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828245 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828220.290783:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c692c0 x1874672572518400/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828245 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828220.290788:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828220.290790:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787828220.290792:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c692c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572518400:192.168.203.129@tcp:38:kworker.0 00010000:00010000:2.0:1787828221.122397:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787828222.146428:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787828223.170419:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787828224.194377:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787828225.218414:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787828225.410321:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828225.410331:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787828225.410335:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828225.410345:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 1177 00000100:00080000:0.0:1787828225.410355:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828225.410361:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:0.0:1787828225.410394:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c6a580 x1874672572518528/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828225.410422:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828225.410426:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787828225.410428:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828225.410432:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 1177 00000100:00080000:0.0:1787828225.410436:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828225.410439:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:0.0:1787828225.410449:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787828225.410462:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c6a1c0 x1874672572518656/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787828225.410465:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c6a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572518528:192.168.203.129@tcp:250:kworker.0 00000100:00080000:0.0:1787828225.410481:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828225.410485:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1787828225.410487:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828225.410513:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 1177 00000100:00080000:0.0:1787828225.410518:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000800:00000100:1.0:1787828225.410523:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00080000:0.0:1787828225.410527:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:0.0:1787828225.410534:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787828225.410548:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c692c0 x1874672572518784/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787828225.410560:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828225/real 1787828225] req@ffff8cb999c6a580 x1874672572518528/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828250 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828225.410574:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c6a580 x1874672572518528/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828250 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828225.410582:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828225.410585:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787828225.410594:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c6a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572518528:192.168.203.129@tcp:250:kworker.0 00000100:00100000:1.0:1787828225.410614:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c6a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572518656:192.168.203.129@tcp:38:kworker.0 00000800:00000100:1.0:1787828225.410640:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787828225.410649:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828225/real 1787828225] req@ffff8cb999c6a1c0 x1874672572518656/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828250 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828225.410657:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c6a1c0 x1874672572518656/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828250 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828225.410663:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828225.410665:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787828225.410667:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c6a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572518656:192.168.203.129@tcp:38:kworker.0 00000100:00100000:1.0:1787828225.410671:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c692c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572518784:192.168.203.129@tcp:38:kworker.0 00000800:00000100:1.0:1787828225.410681:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787828225.410687:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828225/real 1787828225] req@ffff8cb999c692c0 x1874672572518784/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828250 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828225.410694:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c692c0 x1874672572518784/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828250 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828225.410700:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828225.410701:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787828225.410703:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c692c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572518784:192.168.203.129@tcp:38:kworker.0 00010000:00010000:0.0:1787828226.242462:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787828227.266366:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787828228.290423:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787828229.314404:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787828230.338416:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787828230.530387:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828230.530395:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787828230.530398:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828230.530407:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 1182 00000100:00080000:3.0:1787828230.530432:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828230.530438:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:3.0:1787828230.530467:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb992316940 x1874672572518912/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828230.530492:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828230.530495:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787828230.530497:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828230.530499:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 1182 00000100:00080000:3.0:1787828230.530502:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828230.530504:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:3.0:1787828230.530511:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787828230.530518:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb992315e00 x1874672572519040/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828230.530526:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828230.530528:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1787828230.530529:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828230.530532:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 1182 00000100:00080000:3.0:1787828230.530533:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828230.530535:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:3.0:1787828230.530538:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787828230.530544:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb992314f00 x1874672572519168/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787828230.530566:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb992316940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572518912:192.168.203.129@tcp:250:kworker.0 00000800:00000100:1.0:1787828230.530619:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787828230.530659:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828230/real 1787828230] req@ffff8cb992316940 x1874672572518912/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828255 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828230.530672:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb992316940 x1874672572518912/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828255 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828230.530680:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828230.530683:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787828230.530685:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb992316940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572518912:192.168.203.129@tcp:250:kworker.0 00000100:00100000:1.0:1787828230.530690:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb992315e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572519040:192.168.203.129@tcp:38:kworker.0 00000800:00000100:1.0:1787828230.530704:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787828230.530711:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828230/real 1787828230] req@ffff8cb992315e00 x1874672572519040/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828255 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828230.530719:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb992315e00 x1874672572519040/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828255 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828230.530724:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828230.530726:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787828230.530728:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb992315e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572519040:192.168.203.129@tcp:38:kworker.0 00000100:00100000:1.0:1787828230.530731:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb992314f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572519168:192.168.203.129@tcp:38:kworker.0 00000800:00000100:1.0:1787828230.530741:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787828230.530747:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828230/real 1787828230] req@ffff8cb992314f00 x1874672572519168/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828255 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828230.530754:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb992314f00 x1874672572519168/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828255 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828230.530769:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828230.530771:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787828230.530774:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb992314f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572519168:192.168.203.129@tcp:38:kworker.0 00010000:00010000:0.0:1787828231.362417:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787828232.386415:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787828233.410442:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787828234.434398:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787828235.458485:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787828235.650373:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828235.650408:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787828235.650413:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828235.650425:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 1187 00000100:00080000:3.0:1787828235.650439:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828235.650448:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:3.0:1787828235.650490:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb992317480 x1874672572519296/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828235.650528:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828235.650534:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787828235.650537:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828235.650541:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 1187 00000100:00080000:3.0:1787828235.650545:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828235.650549:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:3.0:1787828235.650562:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787828235.650578:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb992314b40 x1874672572519424/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828235.650594:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828235.650603:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00100000:1.0:1787828235.650606:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb992317480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572519296:192.168.203.129@tcp:250:kworker.0 00000100:00080000:3.0:1787828235.650607:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828235.650622:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 1187 00000100:00080000:3.0:1787828235.650628:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828235.650633:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:3.0:1787828235.650653:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787828235.650671:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb992314780 x1874672572519552/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1787828235.650706:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787828235.650772:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828235/real 1787828235] req@ffff8cb992317480 x1874672572519296/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828260 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828235.650799:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb992317480 x1874672572519296/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828260 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828235.650814:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828235.650819:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787828235.650825:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb992317480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572519296:192.168.203.129@tcp:250:kworker.0 00000100:00100000:1.0:1787828235.650858:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb992314b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572519424:192.168.203.129@tcp:38:kworker.0 00000800:00000100:1.0:1787828235.650890:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787828235.650908:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828235/real 1787828235] req@ffff8cb992314b40 x1874672572519424/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828260 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828235.650955:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb992314b40 x1874672572519424/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828260 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828235.650975:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828235.650980:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787828235.650984:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb992314b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572519424:192.168.203.129@tcp:38:kworker.0 00000100:00100000:1.0:1787828235.651005:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb992314780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572519552:192.168.203.129@tcp:38:kworker.0 00000800:00000100:1.0:1787828235.651037:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787828235.651054:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828235/real 1787828235] req@ffff8cb992314780 x1874672572519552/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828260 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828235.651069:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb992314780 x1874672572519552/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828260 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828235.651083:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828235.651090:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787828235.651096:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb992314780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572519552:192.168.203.129@tcp:38:kworker.0 00010000:00010000:2.0:1787828236.482538:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787828237.506464:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787828238.530499:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787828239.554486:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787828240.578411:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787828240.770326:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828240.770335:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787828240.770339:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828240.770347:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 1192 00000100:00080000:3.0:1787828240.770371:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828240.770379:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:3.0:1787828240.770409:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb992316580 x1874672572519680/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828240.770450:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828240.770453:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787828240.770455:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828240.770458:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 1192 00000100:00080000:3.0:1787828240.770462:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828240.770466:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:3.0:1787828240.770486:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787828240.770494:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb992316580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572519680:192.168.203.129@tcp:250:kworker.0 00000100:00100000:3.0:1787828240.770505:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb992314780 x1874672572519808/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828240.770530:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828240.770534:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1787828240.770547:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828240.770552:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 1192 00000100:00080000:3.0:1787828240.770558:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828240.770564:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000800:00000100:2.0:1787828240.770566:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000008:00100000:3.0:1787828240.770581:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787828240.770608:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb992314b40 x1874672572519936/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787828240.770609:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828240/real 1787828240] req@ffff8cb992316580 x1874672572519680/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828265 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828240.770645:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb992316580 x1874672572519680/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828265 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828240.770654:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828240.770657:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787828240.770660:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb992316580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572519680:192.168.203.129@tcp:250:kworker.0 00000100:00100000:2.0:1787828240.770690:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb992314780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572519808:192.168.203.129@tcp:38:kworker.0 00000800:00000100:2.0:1787828240.770719:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787828240.770731:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828240/real 1787828240] req@ffff8cb992314780 x1874672572519808/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828265 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828240.770750:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb992314780 x1874672572519808/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828265 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828240.770756:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828240.770759:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787828240.770761:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb992314780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572519808:192.168.203.129@tcp:38:kworker.0 00000100:00100000:2.0:1787828240.770765:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb992314b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572519936:192.168.203.129@tcp:38:kworker.0 00000800:00000100:2.0:1787828240.770776:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787828240.770782:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828240/real 1787828240] req@ffff8cb992314b40 x1874672572519936/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828265 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828240.770790:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb992314b40 x1874672572519936/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828265 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828240.770796:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828240.770798:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787828240.770800:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb992314b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572519936:192.168.203.129@tcp:38:kworker.0 00010000:00010000:0.0:1787828241.602491:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787828242.626490:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787828243.650453:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787828244.674571:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787828245.698600:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787828245.890481:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828245.890497:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787828245.890503:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828245.890519:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 1198 00000100:00080000:2.0:1787828245.890585:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828245.890597:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:2.0:1787828245.890653:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8fc00 x1874672572520064/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828245.890699:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828245.890705:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787828245.890707:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828245.890712:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 1198 00000100:00080000:2.0:1787828245.890716:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828245.890721:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:2.0:1787828245.890735:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787828245.890751:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8da40 x1874672572520192/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828245.890766:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828245.890773:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1787828245.890779:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1787828245.890780:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8fc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572520064:192.168.203.129@tcp:250:kworker.0 00000100:00080000:2.0:1787828245.890788:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 1198 00000100:00080000:2.0:1787828245.890799:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828245.890804:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:2.0:1787828245.890811:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787828245.890827:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8e1c0 x1874672572520320/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787828245.890999:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8da40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572520192:192.168.203.129@tcp:38:kworker.0 00000100:00100000:1.0:1787828245.891078:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8e1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572520320:192.168.203.129@tcp:38:kworker.0 00010000:00010000:0.0:1787828246.722475:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787828247.746559:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787828248.771168:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:0.0:1787828248.962478:0:2382:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.129:988 00000400:00000100:0.0:1787828248.962511:0:2382:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.129@tcp at host 192.168.203.129 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:0.0:1787828248.962524:0:2382:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.129@tcp 00000800:00000100:0.0:1787828248.962554:0:2382:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.129@tcp 00000800:00000100:0.0:1787828248.962561:0:2382:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.129@tcp 00000100:00100000:1.0:1787828248.962636:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828245/real 1787828248] req@ffff8cb998a8da40 x1874672572520192/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828270 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828248.962663:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8da40 x1874672572520192/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828270 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828248.962675:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828248.962681:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787828248.962686:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8da40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572520192:192.168.203.129@tcp:38:kworker.0 00000100:00100000:1.0:1787828248.962714:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828245/real 1787828248] req@ffff8cb998a8e1c0 x1874672572520320/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828270 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828248.962730:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8e1c0 x1874672572520320/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828270 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828248.962737:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828248.962740:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787828248.962742:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8e1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572520320:192.168.203.129@tcp:38:kworker.0 00000100:00100000:1.0:1787828248.962748:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828245/real 1787828248] req@ffff8cb998a8fc00 x1874672572520064/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828270 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828248.962756:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8fc00 x1874672572520064/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828270 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828248.962761:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828248.962763:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787828248.962765:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8fc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572520064:192.168.203.129@tcp:250:kworker.0 00010000:00010000:0.0:1787828249.794614:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787828250.818523:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787828251.010335:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828251.010357:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787828251.010360:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828251.010369:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 1203 00000100:00080000:2.0:1787828251.010379:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828251.010385:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:2.0:1787828251.010413:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8f0c0 x1874672572520448/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828251.010437:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828251.010439:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787828251.010441:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828251.010443:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 1203 00000100:00080000:2.0:1787828251.010445:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828251.010448:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:2.0:1787828251.010456:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787828251.010464:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8cf00 x1874672572520576/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828251.010476:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828251.010480:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1787828251.010482:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828251.010486:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 1203 00000100:00080000:2.0:1787828251.010490:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828251.010495:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:2.0:1787828251.010501:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787828251.010504:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8f0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572520448:192.168.203.129@tcp:250:kworker.0 00000100:00100000:2.0:1787828251.010515:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8c780 x1874672572520704/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1787828251.010580:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787828251.010619:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828251/real 1787828251] req@ffff8cb998a8f0c0 x1874672572520448/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828276 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828251.010641:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8f0c0 x1874672572520448/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828276 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828251.010665:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828251.010670:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787828251.010675:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8f0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572520448:192.168.203.129@tcp:250:kworker.0 00000100:00100000:1.0:1787828251.010683:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8cf00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572520576:192.168.203.129@tcp:38:kworker.0 00000800:00000100:1.0:1787828251.010707:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787828251.010723:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828251/real 1787828251] req@ffff8cb998a8cf00 x1874672572520576/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828276 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828251.010736:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8cf00 x1874672572520576/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828276 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828251.010745:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828251.010749:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787828251.010753:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8cf00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572520576:192.168.203.129@tcp:38:kworker.0 00000100:00100000:1.0:1787828251.010779:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8c780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572520704:192.168.203.129@tcp:38:kworker.0 00000800:00000100:1.0:1787828251.010801:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787828251.010832:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828251/real 1787828251] req@ffff8cb998a8c780 x1874672572520704/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828276 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828251.010848:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8c780 x1874672572520704/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828276 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828251.010858:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828251.010862:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787828251.010866:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8c780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572520704:192.168.203.129@tcp:38:kworker.0 00010000:00010000:0.0:1787828251.842525:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787828252.866733:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787828253.891602:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787828254.914722:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787828255.938661:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787828256.130360:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828256.130370:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787828256.130375:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828256.130388:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 1208 00000100:00080000:2.0:1787828256.130410:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828256.130417:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:2.0:1787828256.130454:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8e580 x1874672572520832/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828256.130492:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828256.130496:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787828256.130498:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828256.130501:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 1208 00000100:00080000:2.0:1787828256.130504:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828256.130507:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:2.0:1787828256.130515:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787828256.130526:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8d680 x1874672572520960/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828256.130537:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828256.130540:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1787828256.130542:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1787828256.130548:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8e580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572520832:192.168.203.129@tcp:250:kworker.0 00000100:00080000:2.0:1787828256.130549:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 1208 00000100:00080000:2.0:1787828256.130557:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828256.130562:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:2.0:1787828256.130584:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787828256.130597:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8c000 x1874672572521088/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1787828256.130621:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787828256.130685:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828256/real 1787828256] req@ffff8cb998a8e580 x1874672572520832/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828281 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828256.130707:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8e580 x1874672572520832/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828281 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828256.130718:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828256.130721:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787828256.130725:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8e580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572520832:192.168.203.129@tcp:250:kworker.0 00000100:00100000:1.0:1787828256.130751:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8d680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572520960:192.168.203.129@tcp:38:kworker.0 00000800:00000100:1.0:1787828256.130772:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787828256.130807:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828256/real 1787828256] req@ffff8cb998a8d680 x1874672572520960/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828281 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828256.130819:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8d680 x1874672572520960/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828281 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828256.130827:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828256.130830:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787828256.130833:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8d680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572520960:192.168.203.129@tcp:38:kworker.0 00000100:00100000:1.0:1787828256.130838:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572521088:192.168.203.129@tcp:38:kworker.0 00000800:00000100:1.0:1787828256.130856:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787828256.130877:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828256/real 1787828256] req@ffff8cb998a8c000 x1874672572521088/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828281 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828256.130887:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8c000 x1874672572521088/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828281 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828256.130895:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828256.130897:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787828256.130900:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572521088:192.168.203.129@tcp:38:kworker.0 00010000:00010000:0.0:1787828256.962580:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787828257.986431:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787828259.010390:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787828260.034375:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787828261.058388:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787828261.250370:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828261.250378:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787828261.250382:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828261.250389:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 1213 00000100:00080000:0.0:1787828261.250398:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828261.250404:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:0.0:1787828261.250436:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c68000 x1874672572521216/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828261.250459:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828261.250462:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787828261.250464:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828261.250466:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 1213 00000100:00080000:0.0:1787828261.250469:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828261.250472:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:0.0:1787828261.250480:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787828261.250488:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c69680 x1874672572521344/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828261.250496:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1787828261.250501:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c68000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572521216:192.168.203.129@tcp:250:kworker.0 00000100:00080000:0.0:1787828261.250514:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1787828261.250516:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828261.250520:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 1213 00000100:00080000:0.0:1787828261.250524:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828261.250527:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:0.0:1787828261.250532:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787828261.250541:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c6b480 x1874672572521472/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1787828261.250560:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787828261.250622:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828261/real 1787828261] req@ffff8cb999c68000 x1874672572521216/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828286 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828261.250639:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c68000 x1874672572521216/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828286 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828261.250648:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828261.250651:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787828261.250655:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c68000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572521216:192.168.203.129@tcp:250:kworker.0 00000100:00100000:1.0:1787828261.250677:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c69680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572521344:192.168.203.129@tcp:38:kworker.0 00000800:00000100:1.0:1787828261.250716:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787828261.250726:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828261/real 1787828261] req@ffff8cb999c69680 x1874672572521344/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828286 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828261.250735:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c69680 x1874672572521344/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828286 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828261.250765:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828261.250768:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787828261.250771:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c69680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572521344:192.168.203.129@tcp:38:kworker.0 00000100:00100000:1.0:1787828261.250776:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c6b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572521472:192.168.203.129@tcp:38:kworker.0 00000800:00000100:1.0:1787828261.250796:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787828261.250818:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828261/real 1787828261] req@ffff8cb999c6b480 x1874672572521472/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828286 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828261.250831:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c6b480 x1874672572521472/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828286 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828261.250841:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828261.250844:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787828261.250848:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c6b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572521472:192.168.203.129@tcp:38:kworker.0 00010000:00010000:0.0:1787828262.082473:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787828263.106397:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787828264.130478:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787828265.154455:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787828266.178549:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787828266.370346:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828266.370357:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787828266.370361:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828266.370371:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 1218 00000100:00080000:0.0:1787828266.370383:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828266.370390:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:0.0:1787828266.370430:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c68f00 x1874672572521600/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828266.370460:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828266.370464:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787828266.370466:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828266.370470:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 1218 00000100:00080000:0.0:1787828266.370473:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828266.370477:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:0.0:1787828266.370500:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787828266.370516:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c69e00 x1874672572521728/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828266.370531:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828266.370534:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1787828266.370537:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828266.370544:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 1218 00000100:00100000:1.0:1787828266.370546:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c68f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572521600:192.168.203.129@tcp:250:kworker.0 00000100:00080000:0.0:1787828266.370548:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828266.370557:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:0.0:1787828266.370565:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787828266.370577:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c6bc00 x1874672572521856/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1787828266.370630:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787828266.370671:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828266/real 1787828266] req@ffff8cb999c68f00 x1874672572521600/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828291 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828266.370689:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c68f00 x1874672572521600/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828291 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828266.370704:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828266.370737:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787828266.370744:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c68f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572521600:192.168.203.129@tcp:250:kworker.0 00000100:00100000:1.0:1787828266.370793:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c69e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572521728:192.168.203.129@tcp:38:kworker.0 00000800:00000100:1.0:1787828266.370823:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787828266.370837:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828266/real 1787828266] req@ffff8cb999c69e00 x1874672572521728/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828291 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828266.370848:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c69e00 x1874672572521728/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828291 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828266.370856:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828266.370859:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787828266.370862:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c69e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572521728:192.168.203.129@tcp:38:kworker.0 00000100:00100000:1.0:1787828266.370868:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c6bc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572521856:192.168.203.129@tcp:38:kworker.0 00000800:00000100:1.0:1787828266.370883:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787828266.370892:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828266/real 1787828266] req@ffff8cb999c6bc00 x1874672572521856/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828291 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828266.370902:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c6bc00 x1874672572521856/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828291 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828266.370909:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828266.370911:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787828266.370914:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c6bc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572521856:192.168.203.129@tcp:38:kworker.0 00010000:00010000:0.0:1787828267.202495:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787828268.226526:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787828269.250556:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787828270.274535:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787828271.298561:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787828271.490488:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828271.490505:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787828271.490511:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828271.490524:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 1223 00000100:00080000:2.0:1787828271.490537:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828271.490546:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:2.0:1787828271.490591:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8f480 x1874672572521984/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828271.490632:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828271.490637:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787828271.490639:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828271.490644:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 1223 00000100:00080000:2.0:1787828271.490647:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828271.490651:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:2.0:1787828271.490663:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787828271.490704:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8cb40 x1874672572522112/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828271.490719:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828271.490724:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1787828271.490726:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828271.490732:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 1223 00000100:00100000:1.0:1787828271.490761:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572521984:192.168.203.129@tcp:250:kworker.0 00000100:00080000:2.0:1787828271.490761:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828271.490779:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:2.0:1787828271.490789:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787828271.490815:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8e940 x1874672572522240/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1787828271.490901:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787828271.490988:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828271/real 1787828271] req@ffff8cb998a8f480 x1874672572521984/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828296 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828271.491018:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8f480 x1874672572521984/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828296 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828271.491035:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828271.491041:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787828271.491047:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572521984:192.168.203.129@tcp:250:kworker.0 00000100:00100000:1.0:1787828271.491058:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8cb40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572522112:192.168.203.129@tcp:38:kworker.0 00000800:00000100:1.0:1787828271.491093:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787828271.491115:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828271/real 1787828271] req@ffff8cb998a8cb40 x1874672572522112/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828296 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828271.491141:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8cb40 x1874672572522112/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828296 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828271.491158:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828271.491165:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787828271.491173:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8cb40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572522112:192.168.203.129@tcp:38:kworker.0 00000100:00100000:1.0:1787828271.491223:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8e940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572522240:192.168.203.129@tcp:38:kworker.0 00000800:00000100:1.0:1787828271.491255:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787828271.491321:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828271/real 1787828271] req@ffff8cb998a8e940 x1874672572522240/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828296 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828271.491386:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8e940 x1874672572522240/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828296 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828271.491404:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828271.491412:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787828271.491419:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8e940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572522240:192.168.203.129@tcp:38:kworker.0 00010000:00010000:1.0:1787828272.322541:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787828273.346446:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787828274.370501:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787828275.394461:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787828276.418382:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787828276.610358:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828276.610369:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787828276.610374:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828276.610384:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 1228 00000100:00080000:0.0:1787828276.610394:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828276.610402:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:0.0:1787828276.610450:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c6b840 x1874672572522368/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828276.610500:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828276.610505:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787828276.610508:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828276.610515:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 1228 00000100:00080000:0.0:1787828276.610520:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828276.610526:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:3.0:1787828276.610533:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c6b840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572522368:192.168.203.129@tcp:250:kworker.0 00000008:00100000:0.0:1787828276.610540:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787828276.610559:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c69a40 x1874672572522496/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828276.610572:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828276.610576:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1787828276.610579:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828276.610583:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 1228 00000100:00080000:0.0:1787828276.610587:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828276.610590:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:0.0:1787828276.610596:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000800:00000100:3.0:1787828276.610597:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787828276.610613:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c6bc00 x1874672572522624/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1787828276.610720:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828276/real 1787828276] req@ffff8cb999c6b840 x1874672572522368/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828301 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828276.610829:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c6b840 x1874672572522368/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828301 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828276.610842:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828276.610847:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:3.0:1787828276.610852:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c6b840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572522368:192.168.203.129@tcp:250:kworker.0 00000100:00100000:3.0:1787828276.610929:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c69a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572522496:192.168.203.129@tcp:38:kworker.0 00000800:00000100:3.0:1787828276.610950:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:3.0:1787828276.610960:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828276/real 1787828276] req@ffff8cb999c69a40 x1874672572522496/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828301 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828276.610970:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c69a40 x1874672572522496/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828301 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828276.610976:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828276.610979:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:3.0:1787828276.610981:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c69a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572522496:192.168.203.129@tcp:38:kworker.0 00000100:00100000:3.0:1787828276.610985:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c6bc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572522624:192.168.203.129@tcp:38:kworker.0 00000800:00000100:3.0:1787828276.611004:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:3.0:1787828276.611012:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828276/real 1787828276] req@ffff8cb999c6bc00 x1874672572522624/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828301 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828276.611020:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c6bc00 x1874672572522624/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828301 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828276.611025:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828276.611027:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:3.0:1787828276.611028:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c6bc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572522624:192.168.203.129@tcp:38:kworker.0 00010000:00010000:0.0:1787828277.442403:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787828278.466428:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787828279.490392:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787828280.514385:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787828281.538409:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787828281.730323:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828281.730331:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787828281.730334:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828281.730353:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 1233 00000100:00080000:0.0:1787828281.730362:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828281.730368:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:0.0:1787828281.730396:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c69e00 x1874672572522752/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828281.730417:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828281.730420:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787828281.730422:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828281.730425:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 1233 00000100:00080000:0.0:1787828281.730427:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828281.730429:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:0.0:1787828281.730436:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787828281.730447:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c68f00 x1874672572522880/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828281.730456:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828281.730458:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1787828281.730460:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828281.730462:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 1233 00000100:00080000:0.0:1787828281.730465:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828281.730471:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:3.0:1787828281.730473:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c69e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572522752:192.168.203.129@tcp:250:kworker.0 00000008:00100000:0.0:1787828281.730475:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787828281.730485:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c6b480 x1874672572523008/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1787828281.730529:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:3.0:1787828281.730560:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828281/real 1787828281] req@ffff8cb999c69e00 x1874672572522752/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828306 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828281.730574:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c69e00 x1874672572522752/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828306 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828281.730581:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828281.730592:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:3.0:1787828281.730595:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c69e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572522752:192.168.203.129@tcp:250:kworker.0 00000100:00100000:3.0:1787828281.730631:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c68f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572522880:192.168.203.129@tcp:38:kworker.0 00000800:00000100:3.0:1787828281.730650:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:3.0:1787828281.730659:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828281/real 1787828281] req@ffff8cb999c68f00 x1874672572522880/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828306 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828281.730679:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c68f00 x1874672572522880/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828306 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828281.730685:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828281.730688:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:3.0:1787828281.730690:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c68f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572522880:192.168.203.129@tcp:38:kworker.0 00000100:00100000:3.0:1787828281.730694:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c6b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572523008:192.168.203.129@tcp:38:kworker.0 00000800:00000100:3.0:1787828281.730708:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:3.0:1787828281.730715:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828281/real 1787828281] req@ffff8cb999c6b480 x1874672572523008/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828306 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828281.730723:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c6b480 x1874672572523008/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828306 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828281.730728:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828281.730737:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:3.0:1787828281.730740:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c6b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572523008:192.168.203.129@tcp:38:kworker.0 00010000:00010000:1.0:1787828282.562411:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787828283.586411:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787828284.610391:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787828285.634402:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787828286.658367:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787828286.850345:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828286.850353:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787828286.850357:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828286.850365:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 1239 00000100:00080000:2.0:1787828286.850374:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828286.850380:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:2.0:1787828286.850414:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8ed00 x1874672572523136/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828286.850434:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828286.850437:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787828286.850438:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828286.850441:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 1239 00000100:00080000:2.0:1787828286.850443:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828286.850445:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:2.0:1787828286.850454:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787828286.850463:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8f840 x1874672572523264/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828286.850488:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1787828286.850490:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8ed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572523136:192.168.203.129@tcp:250:kworker.0 00000100:00080000:2.0:1787828286.850503:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1787828286.850505:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828286.850509:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 1239 00000100:00080000:2.0:1787828286.850513:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828286.850516:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:2.0:1787828286.850520:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787828286.850530:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8e940 x1874672572523392/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1787828286.850547:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:3.0:1787828286.850620:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828286/real 1787828286] req@ffff8cb998a8ed00 x1874672572523136/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828311 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828286.850636:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8ed00 x1874672572523136/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828311 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828286.850661:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828286.850664:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:3.0:1787828286.850668:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8ed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572523136:192.168.203.129@tcp:250:kworker.0 00000100:00100000:3.0:1787828286.850688:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8f840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572523264:192.168.203.129@tcp:38:kworker.0 00000800:00000100:3.0:1787828286.850707:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:3.0:1787828286.850717:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828286/real 1787828286] req@ffff8cb998a8f840 x1874672572523264/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828311 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828286.850725:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8f840 x1874672572523264/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828311 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828286.850731:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828286.850733:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:3.0:1787828286.850736:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8f840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572523264:192.168.203.129@tcp:38:kworker.0 00000100:00100000:3.0:1787828286.850739:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8e940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572523392:192.168.203.129@tcp:38:kworker.0 00000800:00000100:3.0:1787828286.850750:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:3.0:1787828286.850756:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828286/real 1787828286] req@ffff8cb998a8e940 x1874672572523392/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828311 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828286.850764:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8e940 x1874672572523392/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828311 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828286.850769:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828286.850771:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:3.0:1787828286.850773:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8e940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572523392:192.168.203.129@tcp:38:kworker.0 00010000:00010000:1.0:1787828287.682366:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787828288.706417:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787828289.730345:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787828290.754411:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787828291.778408:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787828291.970325:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828291.970333:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787828291.970337:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828291.970344:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 1244 00000100:00080000:2.0:1787828291.970353:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828291.970358:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:2.0:1787828291.970387:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8cb40 x1874672572523520/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828291.970408:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828291.970411:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787828291.970413:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828291.970415:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 1244 00000100:00080000:2.0:1787828291.970418:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828291.970420:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:2.0:1787828291.970427:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787828291.970434:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8f480 x1874672572523648/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828291.970452:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828291.970454:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1787828291.970456:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828291.970459:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 1244 00000100:00080000:2.0:1787828291.970462:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828291.970465:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:2.0:1787828291.970469:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787828291.970476:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8c000 x1874672572523776/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1787828291.970483:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8cb40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572523520:192.168.203.129@tcp:250:kworker.0 00000800:00000100:3.0:1787828291.970556:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:3.0:1787828291.970604:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828291/real 1787828291] req@ffff8cb998a8cb40 x1874672572523520/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828316 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828291.970624:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8cb40 x1874672572523520/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828316 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828291.970637:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828291.970640:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:3.0:1787828291.970643:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8cb40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572523520:192.168.203.129@tcp:250:kworker.0 00000100:00100000:3.0:1787828291.970649:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572523648:192.168.203.129@tcp:38:kworker.0 00000800:00000100:3.0:1787828291.970667:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:3.0:1787828291.970676:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828291/real 1787828291] req@ffff8cb998a8f480 x1874672572523648/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828316 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828291.970685:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8f480 x1874672572523648/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828316 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828291.970690:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828291.970693:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:3.0:1787828291.970695:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572523648:192.168.203.129@tcp:38:kworker.0 00000100:00100000:3.0:1787828291.970712:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572523776:192.168.203.129@tcp:38:kworker.0 00000800:00000100:3.0:1787828291.970723:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:3.0:1787828291.970729:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828291/real 1787828291] req@ffff8cb998a8c000 x1874672572523776/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828316 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828291.970746:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8c000 x1874672572523776/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828316 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828291.970752:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828291.970754:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:3.0:1787828291.970756:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572523776:192.168.203.129@tcp:38:kworker.0 00010000:00010000:0.0:1787828292.802413:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787828293.826384:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787828294.850390:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787828295.874417:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787828296.898398:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787828297.090336:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828297.090344:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787828297.090348:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828297.090356:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 1249 00000100:00080000:2.0:1787828297.090365:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828297.090371:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:2.0:1787828297.090404:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8d680 x1874672572523904/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828297.090426:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828297.090429:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787828297.090430:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828297.090433:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 1249 00000100:00080000:2.0:1787828297.090435:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828297.090437:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:2.0:1787828297.090459:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787828297.090468:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8e580 x1874672572524032/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828297.090477:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828297.090482:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00100000:3.0:1787828297.090483:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8d680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572523904:192.168.203.129@tcp:250:kworker.0 00000100:00080000:2.0:1787828297.090484:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828297.090492:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 1249 00000100:00080000:2.0:1787828297.090509:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828297.090513:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:2.0:1787828297.090517:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787828297.090527:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8c780 x1874672572524160/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1787828297.090566:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:3.0:1787828297.090619:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828297/real 1787828297] req@ffff8cb998a8d680 x1874672572523904/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828322 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828297.090634:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8d680 x1874672572523904/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828322 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828297.090641:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828297.090644:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:3.0:1787828297.090649:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8d680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572523904:192.168.203.129@tcp:250:kworker.0 00000100:00100000:3.0:1787828297.090669:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8e580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572524032:192.168.203.129@tcp:38:kworker.0 00000800:00000100:3.0:1787828297.090686:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:3.0:1787828297.090694:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828297/real 1787828297] req@ffff8cb998a8e580 x1874672572524032/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828322 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828297.090703:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8e580 x1874672572524032/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828322 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828297.090719:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828297.090722:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:3.0:1787828297.090724:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8e580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572524032:192.168.203.129@tcp:38:kworker.0 00000100:00100000:3.0:1787828297.090728:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8c780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572524160:192.168.203.129@tcp:38:kworker.0 00000800:00000100:3.0:1787828297.090742:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:3.0:1787828297.090749:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828297/real 1787828297] req@ffff8cb998a8c780 x1874672572524160/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828322 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828297.090756:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8c780 x1874672572524160/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828322 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828297.090762:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828297.090763:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:3.0:1787828297.090765:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8c780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572524160:192.168.203.129@tcp:38:kworker.0 00010000:00010000:1.0:1787828297.922404:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787828298.946423:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787828299.970396:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787828300.994434:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787828302.018420:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787828302.210347:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828302.210355:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787828302.210359:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828302.210367:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 1254 00000100:00080000:2.0:1787828302.210377:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828302.210382:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:2.0:1787828302.210414:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8cf00 x1874672572524288/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828302.210436:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828302.210439:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787828302.210440:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828302.210443:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 1254 00000100:00080000:2.0:1787828302.210470:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828302.210477:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:2.0:1787828302.210485:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787828302.210493:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8cf00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572524288:192.168.203.129@tcp:250:kworker.0 00000100:00100000:2.0:1787828302.210499:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8f0c0 x1874672572524416/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828302.210509:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828302.210511:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1787828302.210513:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828302.210516:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 1254 00000100:00080000:2.0:1787828302.210519:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828302.210521:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:2.0:1787828302.210524:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787828302.210534:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8fc00 x1874672572524544/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1787828302.210552:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:3.0:1787828302.210595:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828302/real 1787828302] req@ffff8cb998a8cf00 x1874672572524288/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828327 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828302.210610:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8cf00 x1874672572524288/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828327 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828302.210618:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828302.210621:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:3.0:1787828302.210631:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8cf00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572524288:192.168.203.129@tcp:250:kworker.0 00000100:00100000:3.0:1787828302.210652:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8f0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572524416:192.168.203.129@tcp:38:kworker.0 00000800:00000100:3.0:1787828302.210669:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:3.0:1787828302.210678:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828302/real 1787828302] req@ffff8cb998a8f0c0 x1874672572524416/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828327 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828302.210686:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8f0c0 x1874672572524416/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828327 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828302.210692:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828302.210694:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:3.0:1787828302.210696:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8f0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572524416:192.168.203.129@tcp:38:kworker.0 00000100:00100000:3.0:1787828302.210700:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8fc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572524544:192.168.203.129@tcp:38:kworker.0 00000800:00000100:3.0:1787828302.210709:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:3.0:1787828302.210717:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828302/real 1787828302] req@ffff8cb998a8fc00 x1874672572524544/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828327 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828302.210729:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8fc00 x1874672572524544/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828327 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828302.210738:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828302.210742:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:3.0:1787828302.210746:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8fc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572524544:192.168.203.129@tcp:38:kworker.0 00010000:00010000:1.0:1787828303.042434:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787828304.066424:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787828305.090456:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787828306.114456:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787828307.138542:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787828307.330386:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828307.330397:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787828307.330401:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828307.330410:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 1259 00000100:00080000:2.0:1787828307.330420:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828307.330440:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:2.0:1787828307.330479:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8e1c0 x1874672572524672/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828307.330507:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828307.330528:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787828307.330531:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828307.330537:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 1259 00000100:00080000:2.0:1787828307.330543:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828307.330550:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:2.0:1787828307.330560:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787828307.330561:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8e1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572524672:192.168.203.129@tcp:250:kworker.0 00000100:00100000:2.0:1787828307.330582:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8da40 x1874672572524800/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828307.330597:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828307.330600:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1787828307.330602:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828307.330607:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 1259 00000100:00080000:2.0:1787828307.330612:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828307.330615:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:2.0:1787828307.330620:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787828307.330632:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8d2c0 x1874672572524928/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1787828307.330653:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:3.0:1787828307.330724:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828307/real 1787828307] req@ffff8cb998a8e1c0 x1874672572524672/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828332 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828307.330745:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8e1c0 x1874672572524672/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828332 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828307.330755:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828307.330758:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:3.0:1787828307.330762:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8e1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572524672:192.168.203.129@tcp:250:kworker.0 00000100:00100000:3.0:1787828307.330787:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8da40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572524800:192.168.203.129@tcp:38:kworker.0 00000800:00000100:3.0:1787828307.330807:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:3.0:1787828307.330817:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828307/real 1787828307] req@ffff8cb998a8da40 x1874672572524800/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828332 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828307.330830:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8da40 x1874672572524800/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828332 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828307.330838:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828307.330841:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:3.0:1787828307.330845:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8da40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572524800:192.168.203.129@tcp:38:kworker.0 00000100:00100000:3.0:1787828307.330852:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8d2c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572524928:192.168.203.129@tcp:38:kworker.0 00000800:00000100:3.0:1787828307.330882:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:3.0:1787828307.330892:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828307/real 1787828307] req@ffff8cb998a8d2c0 x1874672572524928/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828332 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828307.330901:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8d2c0 x1874672572524928/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828332 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828307.330915:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828307.330918:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:3.0:1787828307.330920:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8d2c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572524928:192.168.203.129@tcp:38:kworker.0 00010000:00010000:1.0:1787828308.162472:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787828309.186543:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787828310.210593:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787828311.234539:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787828312.258485:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787828312.450418:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828312.450431:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787828312.450437:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828312.450450:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 1264 00000100:00080000:2.0:1787828312.450464:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828312.450474:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:2.0:1787828312.450536:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8de00 x1874672572525056/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828312.450581:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828312.450588:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787828312.450590:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828312.450598:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 1264 00000100:00080000:2.0:1787828312.450604:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828312.450609:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:2.0:1787828312.450622:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787828312.450629:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8de00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572525056:192.168.203.129@tcp:250:kworker.0 00000100:00100000:2.0:1787828312.450647:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8c3c0 x1874672572525184/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828312.450664:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828312.450669:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1787828312.450672:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828312.450678:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 1264 00000100:00080000:2.0:1787828312.450682:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828312.450686:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:2.0:1787828312.450692:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787828312.450709:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8d2c0 x1874672572525312/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1787828312.450884:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8c3c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572525184:192.168.203.129@tcp:38:kworker.0 00000100:00100000:3.0:1787828312.452217:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8d2c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572525312:192.168.203.129@tcp:38:kworker.0 00010000:00010000:1.0:1787828313.282523:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787828314.306497:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787828315.330848:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:3.0:1787828315.522729:0:2383:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.129:988 00000400:00000100:3.0:1787828315.522805:0:2383:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.129@tcp at host 192.168.203.129 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:3.0:1787828315.522836:0:2383:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.129@tcp 00000800:00000100:3.0:1787828315.522914:0:2383:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.129@tcp 00000800:00000100:3.0:1787828315.522932:0:2383:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.129@tcp 00000100:00100000:1.0:1787828315.522980:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828312/real 1787828315] req@ffff8cb998a8c3c0 x1874672572525184/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828337 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828315.523033:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8c3c0 x1874672572525184/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828337 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828315.523055:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828315.523064:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787828315.523072:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8c3c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572525184:192.168.203.129@tcp:38:kworker.0 00000100:00100000:1.0:1787828315.523086:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828312/real 1787828315] req@ffff8cb998a8d2c0 x1874672572525312/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828337 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828315.523110:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8d2c0 x1874672572525312/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828337 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828315.523138:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828315.523146:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787828315.523153:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8d2c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572525312:192.168.203.129@tcp:38:kworker.0 00000100:00100000:1.0:1787828315.523198:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828312/real 1787828315] req@ffff8cb998a8de00 x1874672572525056/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828337 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828315.523227:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8de00 x1874672572525056/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828337 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828315.523243:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828315.523249:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787828315.523255:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8de00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572525056:192.168.203.129@tcp:250:kworker.0 00010000:00010000:0.0:1787828316.354883:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787828317.379256:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787828317.570382:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828317.570409:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787828317.570414:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828317.570427:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 1269 00000100:00080000:0.0:1787828317.570439:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828317.570450:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:0.0:1787828317.570493:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c69680 x1874672572525440/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828317.570530:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828317.570535:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787828317.570538:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828317.570545:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 1269 00000100:00080000:0.0:1787828317.570552:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828317.570558:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:0.0:1787828317.570570:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787828317.570571:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c69680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572525440:192.168.203.129@tcp:250:kworker.0 00000100:00100000:0.0:1787828317.570594:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c68000 x1874672572525568/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828317.570613:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:1.0:1787828317.570636:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00080000:0.0:1787828317.570646:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1787828317.570650:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828317.570671:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 1269 00000100:00080000:0.0:1787828317.570678:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828317.570683:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:0.0:1787828317.570689:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787828317.570702:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828317/real 1787828317] req@ffff8cb999c69680 x1874672572525440/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828342 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828317.570710:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c692c0 x1874672572525696/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787828317.570728:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c69680 x1874672572525440/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828342 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828317.570740:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828317.570745:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787828317.570750:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c69680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572525440:192.168.203.129@tcp:250:kworker.0 00000100:00100000:1.0:1787828317.570773:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c68000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572525568:192.168.203.129@tcp:38:kworker.0 00000800:00000100:1.0:1787828317.570796:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787828317.570810:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828317/real 1787828317] req@ffff8cb999c68000 x1874672572525568/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828342 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828317.570820:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c68000 x1874672572525568/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828342 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828317.570837:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828317.570840:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787828317.570844:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c68000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572525568:192.168.203.129@tcp:38:kworker.0 00000100:00100000:1.0:1787828317.570850:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c692c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572525696:192.168.203.129@tcp:38:kworker.0 00000800:00000100:1.0:1787828317.570872:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787828317.570884:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828317/real 1787828317] req@ffff8cb999c692c0 x1874672572525696/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828342 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828317.570897:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c692c0 x1874672572525696/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828342 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828317.570905:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828317.570909:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787828317.570912:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c692c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572525696:192.168.203.129@tcp:38:kworker.0 00010000:00010000:2.0:1787828318.402808:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787828319.426430:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787828320.451104:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787828321.474578:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787828322.498529:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787828322.690345:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828322.690354:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787828322.690369:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828322.690377:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 1274 00000100:00080000:0.0:1787828322.690385:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828322.690391:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:0.0:1787828322.690421:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c6a1c0 x1874672572525824/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828322.690443:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828322.690446:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787828322.690448:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828322.690452:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 1274 00000100:00080000:0.0:1787828322.690456:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828322.690461:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:0.0:1787828322.690471:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787828322.690486:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c6a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572525824:192.168.203.129@tcp:250:kworker.0 00000100:00100000:0.0:1787828322.690487:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c6a580 x1874672572525952/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828322.690517:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828322.690520:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1787828322.690522:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828322.690527:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 1274 00000100:00080000:0.0:1787828322.690533:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828322.690536:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:0.0:1787828322.690541:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787828322.690553:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c692c0 x1874672572526080/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1787828322.690559:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787828322.690610:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828322/real 1787828322] req@ffff8cb999c6a1c0 x1874672572525824/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828347 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828322.690623:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c6a1c0 x1874672572525824/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828347 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828322.690642:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828322.690646:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787828322.690649:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c6a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572525824:192.168.203.129@tcp:250:kworker.0 00000100:00100000:1.0:1787828322.690667:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c6a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572525952:192.168.203.129@tcp:38:kworker.0 00000800:00000100:1.0:1787828322.690692:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787828322.690707:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828322/real 1787828322] req@ffff8cb999c6a580 x1874672572525952/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828347 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828322.690721:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c6a580 x1874672572525952/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828347 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828322.690729:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828322.690733:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787828322.690736:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c6a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572525952:192.168.203.129@tcp:38:kworker.0 00000100:00100000:1.0:1787828322.690742:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c692c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572526080:192.168.203.129@tcp:38:kworker.0 00000800:00000100:1.0:1787828322.690763:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787828322.690775:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828322/real 1787828322] req@ffff8cb999c692c0 x1874672572526080/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828347 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828322.690787:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c692c0 x1874672572526080/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828347 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828322.690795:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828322.690799:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787828322.690802:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c692c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572526080:192.168.203.129@tcp:38:kworker.0 00010000:00010000:2.0:1787828323.523048:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787828324.546641:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787828325.570432:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787828326.594418:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787828327.618666:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787828327.810461:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828327.810475:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787828327.810481:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828327.810495:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 1279 00000100:00080000:3.0:1787828327.810512:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828327.810521:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:3.0:1787828327.810582:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb992317480 x1874672572526208/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828327.810627:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828327.810632:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787828327.810658:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828327.810662:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 1279 00000100:00080000:3.0:1787828327.810667:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828327.810673:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:3.0:1787828327.810687:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787828327.810704:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb992314f00 x1874672572526336/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828327.810717:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828327.810722:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00100000:1.0:1787828327.810723:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb992317480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572526208:192.168.203.129@tcp:250:kworker.0 00000100:00080000:3.0:1787828327.810726:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828327.810738:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 1279 00000100:00080000:3.0:1787828327.810742:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828327.810745:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:3.0:1787828327.810750:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787828327.810764:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb992315e00 x1874672572526464/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1787828327.810847:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787828327.810958:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828327/real 1787828327] req@ffff8cb992317480 x1874672572526208/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828352 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828327.810987:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb992317480 x1874672572526208/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828352 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828327.811011:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828327.811017:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787828327.811024:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb992317480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572526208:192.168.203.129@tcp:250:kworker.0 00000100:00100000:1.0:1787828327.811066:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb992314f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572526336:192.168.203.129@tcp:38:kworker.0 00000800:00000100:1.0:1787828327.811102:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787828327.811137:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828327/real 1787828327] req@ffff8cb992314f00 x1874672572526336/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828352 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828327.811158:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb992314f00 x1874672572526336/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828352 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828327.811171:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828327.811177:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787828327.811183:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb992314f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572526336:192.168.203.129@tcp:38:kworker.0 00000100:00100000:1.0:1787828327.811194:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb992315e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572526464:192.168.203.129@tcp:38:kworker.0 00000800:00000100:1.0:1787828327.811266:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787828327.811389:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828327/real 1787828327] req@ffff8cb992315e00 x1874672572526464/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828352 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828327.811408:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb992315e00 x1874672572526464/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828352 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828327.811418:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828327.811422:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787828327.811426:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb992315e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572526464:192.168.203.129@tcp:38:kworker.0 00010000:00010000:1.0:1787828328.642445:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787828329.666380:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787828330.690490:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787828331.714427:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787828332.738517:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787828332.930354:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828332.930363:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787828332.930367:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828332.930376:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 1285 00000100:00080000:3.0:1787828332.930386:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828332.930391:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:3.0:1787828332.930424:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb992316940 x1874672572526592/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828332.930468:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828332.930488:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787828332.930491:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828332.930495:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 1285 00000100:00080000:3.0:1787828332.930501:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828332.930507:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:3.0:1787828332.930517:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787828332.930521:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb992316940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572526592:192.168.203.129@tcp:250:kworker.0 00000100:00100000:3.0:1787828332.930533:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb9923161c0 x1874672572526720/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828332.930543:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828332.930546:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1787828332.930548:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828332.930550:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 1285 00000100:00080000:3.0:1787828332.930554:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828332.930557:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:3.0:1787828332.930561:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787828332.930568:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb992315a40 x1874672572526848/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1787828332.930579:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787828332.930613:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828332/real 1787828332] req@ffff8cb992316940 x1874672572526592/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828357 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828332.930626:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb992316940 x1874672572526592/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828357 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828332.930635:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828332.930638:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787828332.930641:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb992316940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572526592:192.168.203.129@tcp:250:kworker.0 00000100:00100000:1.0:1787828332.930659:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb9923161c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572526720:192.168.203.129@tcp:38:kworker.0 00000800:00000100:1.0:1787828332.930684:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787828332.930693:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828332/real 1787828332] req@ffff8cb9923161c0 x1874672572526720/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828357 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828332.930701:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb9923161c0 x1874672572526720/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828357 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828332.930707:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828332.930710:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787828332.930712:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb9923161c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572526720:192.168.203.129@tcp:38:kworker.0 00000100:00100000:1.0:1787828332.930716:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb992315a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572526848:192.168.203.129@tcp:38:kworker.0 00000800:00000100:1.0:1787828332.930726:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787828332.930732:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828332/real 1787828332] req@ffff8cb992315a40 x1874672572526848/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828357 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828332.930740:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb992315a40 x1874672572526848/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828357 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828332.930745:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828332.930747:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787828332.930749:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb992315a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572526848:192.168.203.129@tcp:38:kworker.0 00010000:00010000:0.0:1787828333.762406:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787828334.786461:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787828335.810528:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787828336.834575:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787828337.858526:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787828338.050454:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828338.050465:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787828338.050470:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828338.050481:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 1290 00000100:00080000:2.0:1787828338.050493:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828338.050502:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:2.0:1787828338.050541:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8da40 x1874672572526976/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828338.050577:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828338.050582:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787828338.050584:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828338.050589:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 1290 00000100:00080000:2.0:1787828338.050593:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828338.050597:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:2.0:1787828338.050608:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787828338.050621:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8e1c0 x1874672572527104/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828338.050633:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828338.050638:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1787828338.050640:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828338.050645:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 1290 00000100:00080000:2.0:1787828338.050650:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828338.050655:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:2.0:1787828338.050662:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787828338.050696:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8fc00 x1874672572527232/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787828338.050705:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8da40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572526976:192.168.203.129@tcp:250:kworker.0 00000800:00000100:1.0:1787828338.050840:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787828338.050931:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828338/real 1787828338] req@ffff8cb998a8da40 x1874672572526976/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828363 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828338.050966:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8da40 x1874672572526976/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828363 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828338.050987:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828338.050996:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787828338.051004:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8da40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572526976:192.168.203.129@tcp:250:kworker.0 00000100:00100000:1.0:1787828338.051019:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8e1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572527104:192.168.203.129@tcp:38:kworker.0 00000800:00000100:1.0:1787828338.051072:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787828338.051095:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828338/real 1787828338] req@ffff8cb998a8e1c0 x1874672572527104/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828363 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828338.051118:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8e1c0 x1874672572527104/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828363 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828338.051133:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828338.051155:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787828338.051163:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8e1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572527104:192.168.203.129@tcp:38:kworker.0 00000100:00100000:1.0:1787828338.051253:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8fc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572527232:192.168.203.129@tcp:38:kworker.0 00000800:00000100:1.0:1787828338.051322:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787828338.051339:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828338/real 1787828338] req@ffff8cb998a8fc00 x1874672572527232/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828363 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828338.051353:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8fc00 x1874672572527232/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828363 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828338.051378:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828338.051383:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787828338.051387:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8fc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572527232:192.168.203.129@tcp:38:kworker.0 00010000:00010000:1.0:1787828338.882518:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787828339.906570:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787828340.930513:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787828341.954469:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787828342.978376:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787828343.170447:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828343.170455:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787828343.170460:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828343.170471:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 1295 00000100:00080000:2.0:1787828343.170499:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828343.170516:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:2.0:1787828343.170591:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8f0c0 x1874672572527360/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828343.170659:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828343.170667:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787828343.170670:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828343.170676:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 1295 00000100:00080000:2.0:1787828343.170681:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828343.170685:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:2.0:1787828343.170696:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787828343.170701:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8f0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572527360:192.168.203.129@tcp:250:kworker.0 00000100:00100000:2.0:1787828343.170714:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8cf00 x1874672572527488/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828343.170728:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828343.170733:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1787828343.170735:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828343.170740:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 1295 00000100:00080000:2.0:1787828343.170745:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000800:00000100:1.0:1787828343.170761:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00080000:2.0:1787828343.170767:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:2.0:1787828343.170775:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787828343.170788:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8c780 x1874672572527616/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787828343.170821:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828343/real 1787828343] req@ffff8cb998a8f0c0 x1874672572527360/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828368 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828343.170836:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8f0c0 x1874672572527360/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828368 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828343.170844:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828343.170847:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787828343.170850:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8f0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572527360:192.168.203.129@tcp:250:kworker.0 00000100:00100000:1.0:1787828343.170870:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8cf00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572527488:192.168.203.129@tcp:38:kworker.0 00000800:00000100:1.0:1787828343.170888:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787828343.170897:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828343/real 1787828343] req@ffff8cb998a8cf00 x1874672572527488/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828368 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828343.170905:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8cf00 x1874672572527488/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828368 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828343.170921:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828343.170924:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787828343.170926:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8cf00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572527488:192.168.203.129@tcp:38:kworker.0 00000100:00100000:1.0:1787828343.170931:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8c780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572527616:192.168.203.129@tcp:38:kworker.0 00000800:00000100:1.0:1787828343.170945:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787828343.170952:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828343/real 1787828343] req@ffff8cb998a8c780 x1874672572527616/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828368 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828343.170959:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8c780 x1874672572527616/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828368 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828343.170964:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828343.170966:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787828343.170968:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8c780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572527616:192.168.203.129@tcp:38:kworker.0 00010000:00010000:1.0:1787828344.002449:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787828345.026477:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787828346.050502:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787828347.074512:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787828348.098458:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787828348.290425:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828348.290438:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787828348.290443:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828348.290458:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 1300 00000100:00080000:0.0:1787828348.290472:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828348.290482:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:0.0:1787828348.290529:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c68000 x1874672572527744/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828348.290573:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828348.290608:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787828348.290612:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828348.290621:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 1300 00000100:00080000:0.0:1787828348.290629:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828348.290637:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:0.0:1787828348.290652:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787828348.290661:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c68000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572527744:192.168.203.129@tcp:250:kworker.0 00000100:00100000:0.0:1787828348.290688:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c69680 x1874672572527872/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828348.290717:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828348.290725:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1787828348.290730:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828348.290740:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 1300 00000100:00080000:0.0:1787828348.290750:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000800:00000100:3.0:1787828348.290757:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00080000:0.0:1787828348.290768:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:0.0:1787828348.290783:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787828348.290810:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c6b480 x1874672572528000/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1787828348.290830:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828348/real 1787828348] req@ffff8cb999c68000 x1874672572527744/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828373 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828348.290881:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c68000 x1874672572527744/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828373 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828348.290895:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828348.290901:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:3.0:1787828348.290906:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c68000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572527744:192.168.203.129@tcp:250:kworker.0 00000100:00100000:3.0:1787828348.290939:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c69680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572527872:192.168.203.129@tcp:38:kworker.0 00000800:00000100:3.0:1787828348.290970:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:3.0:1787828348.290987:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828348/real 1787828348] req@ffff8cb999c69680 x1874672572527872/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828373 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828348.291002:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c69680 x1874672572527872/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828373 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828348.291013:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828348.291017:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:3.0:1787828348.291021:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c69680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572527872:192.168.203.129@tcp:38:kworker.0 00000100:00100000:3.0:1787828348.291210:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c6b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572528000:192.168.203.129@tcp:38:kworker.0 00000800:00000100:3.0:1787828348.291248:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:3.0:1787828348.291495:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828348/real 1787828348] req@ffff8cb999c6b480 x1874672572528000/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828373 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828348.291516:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c6b480 x1874672572528000/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828373 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828348.291623:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828348.291631:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:3.0:1787828348.291638:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c6b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572528000:192.168.203.129@tcp:38:kworker.0 00010000:00010000:0.0:1787828349.122511:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787828350.146503:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787828351.170418:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787828352.194378:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787828353.218484:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787828353.410442:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828353.410458:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787828353.410465:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828353.410480:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 1305 00000100:00080000:3.0:1787828353.410497:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828353.410510:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:3.0:1787828353.410559:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb9923170c0 x1874672572528128/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828353.410615:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828353.410624:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787828353.410628:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828353.410635:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 1305 00000100:00080000:3.0:1787828353.410643:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828353.410651:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:3.0:1787828353.410668:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787828353.410698:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb992315a40 x1874672572528256/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787828353.410699:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb9923170c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572528128:192.168.203.129@tcp:250:kworker.0 00000100:00080000:3.0:1787828353.410728:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828353.410734:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1787828353.410757:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828353.410764:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 1305 00000100:00080000:3.0:1787828353.410772:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828353.410778:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:3.0:1787828353.410786:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000800:00000100:1.0:1787828353.410795:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:3.0:1787828353.410812:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb9923161c0 x1874672572528384/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787828353.410868:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828353/real 1787828353] req@ffff8cb9923170c0 x1874672572528128/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828378 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828353.410937:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb9923170c0 x1874672572528128/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828378 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828353.410952:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828353.410958:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787828353.410975:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb9923170c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572528128:192.168.203.129@tcp:250:kworker.0 00000100:00100000:1.0:1787828353.411007:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb992315a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572528256:192.168.203.129@tcp:38:kworker.0 00000800:00000100:1.0:1787828353.411042:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787828353.411061:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828353/real 1787828353] req@ffff8cb992315a40 x1874672572528256/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828378 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828353.411076:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb992315a40 x1874672572528256/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828378 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828353.411086:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828353.411091:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787828353.411095:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb992315a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572528256:192.168.203.129@tcp:38:kworker.0 00000100:00100000:1.0:1787828353.411139:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb9923161c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572528384:192.168.203.129@tcp:38:kworker.0 00000800:00000100:1.0:1787828353.411171:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787828353.411187:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828353/real 1787828353] req@ffff8cb9923161c0 x1874672572528384/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828378 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828353.411202:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb9923161c0 x1874672572528384/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828378 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828353.411230:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828353.411234:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787828353.411239:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb9923161c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572528384:192.168.203.129@tcp:38:kworker.0 00010000:00010000:1.0:1787828354.242426:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787828355.266498:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787828356.290425:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787828357.314450:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787828358.338443:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787828358.530387:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828358.530396:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787828358.530400:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828358.530409:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 1310 00000100:00080000:3.0:1787828358.530420:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828358.530427:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:3.0:1787828358.530461:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb992316940 x1874672572528512/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828358.530488:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828358.530493:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787828358.530495:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828358.530500:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 1310 00000100:00080000:3.0:1787828358.530504:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828358.530510:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:3.0:1787828358.530530:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787828358.530539:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb992316940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572528512:192.168.203.129@tcp:250:kworker.0 00000100:00100000:3.0:1787828358.530546:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb992315e00 x1874672572528640/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828358.530576:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828358.530579:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1787828358.530581:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828358.530584:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 1310 00000100:00080000:3.0:1787828358.530589:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828358.530592:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:3.0:1787828358.530597:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787828358.530606:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb992314f00 x1874672572528768/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1787828358.530617:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787828358.530670:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828358/real 1787828358] req@ffff8cb992316940 x1874672572528512/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828383 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828358.530692:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb992316940 x1874672572528512/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828383 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828358.530706:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828358.530711:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787828358.530716:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb992316940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572528512:192.168.203.129@tcp:250:kworker.0 00000100:00100000:1.0:1787828358.530742:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb992315e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572528640:192.168.203.129@tcp:38:kworker.0 00000800:00000100:1.0:1787828358.530774:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787828358.530792:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828358/real 1787828358] req@ffff8cb992315e00 x1874672572528640/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828383 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828358.530808:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb992315e00 x1874672572528640/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828383 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828358.530832:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828358.530837:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787828358.530842:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb992315e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572528640:192.168.203.129@tcp:38:kworker.0 00000100:00100000:1.0:1787828358.530849:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb992314f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572528768:192.168.203.129@tcp:38:kworker.0 00000800:00000100:1.0:1787828358.530879:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787828358.530894:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828358/real 1787828358] req@ffff8cb992314f00 x1874672572528768/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828383 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828358.530909:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb992314f00 x1874672572528768/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828383 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828358.530919:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828358.530923:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787828358.530927:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb992314f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572528768:192.168.203.129@tcp:38:kworker.0 00010000:00010000:1.0:1787828359.362456:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787828360.386406:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787828361.410441:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787828362.434457:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787828363.458398:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787828363.650329:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828363.650337:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787828363.650341:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828363.650349:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 1315 00000100:00080000:3.0:1787828363.650358:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828363.650364:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:3.0:1787828363.650394:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb992317480 x1874672572528896/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828363.650419:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828363.650421:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787828363.650423:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828363.650425:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 1315 00000100:00080000:3.0:1787828363.650428:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828363.650430:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:3.0:1787828363.650449:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787828363.650459:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb992314b40 x1874672572529024/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828363.650469:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1787828363.650472:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb992317480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572528896:192.168.203.129@tcp:250:kworker.0 00000100:00080000:3.0:1787828363.650473:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1787828363.650476:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828363.650484:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 1315 00000100:00080000:3.0:1787828363.650486:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828363.650499:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:3.0:1787828363.650504:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787828363.650517:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb992314780 x1874672572529152/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1787828363.650542:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787828363.650579:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828363/real 1787828363] req@ffff8cb992317480 x1874672572528896/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828388 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828363.650598:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb992317480 x1874672572528896/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828388 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828363.650625:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828363.650629:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787828363.650632:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb992317480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572528896:192.168.203.129@tcp:250:kworker.0 00000100:00100000:1.0:1787828363.650655:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb992314b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572529024:192.168.203.129@tcp:38:kworker.0 00000800:00000100:1.0:1787828363.650679:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787828363.650694:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828363/real 1787828363] req@ffff8cb992314b40 x1874672572529024/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828388 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828363.650708:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb992314b40 x1874672572529024/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828388 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828363.650716:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828363.650718:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787828363.650720:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb992314b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572529024:192.168.203.129@tcp:38:kworker.0 00000100:00100000:1.0:1787828363.650725:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb992314780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572529152:192.168.203.129@tcp:38:kworker.0 00000800:00000100:1.0:1787828363.650741:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787828363.650749:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828363/real 1787828363] req@ffff8cb992314780 x1874672572529152/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828388 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828363.650756:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb992314780 x1874672572529152/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828388 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828363.650762:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828363.650764:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787828363.650767:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb992314780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572529152:192.168.203.129@tcp:38:kworker.0 00010000:00010000:0.0:1787828364.482421:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787828365.506433:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787828366.530483:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787828367.554463:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787828368.578579:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787828368.770327:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828368.770338:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787828368.770342:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828368.770352:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 1320 00000100:00080000:3.0:1787828368.770364:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828368.770371:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:3.0:1787828368.770409:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb992316580 x1874672572529280/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828368.770430:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828368.770433:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787828368.770434:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828368.770437:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 1320 00000100:00080000:3.0:1787828368.770450:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828368.770454:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:3.0:1787828368.770463:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787828368.770476:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb992316580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572529280:192.168.203.129@tcp:250:kworker.0 00000100:00100000:3.0:1787828368.770477:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb992314780 x1874672572529408/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828368.770492:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828368.770494:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1787828368.770495:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828368.770499:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 1320 00000100:00080000:3.0:1787828368.770502:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828368.770504:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:3.0:1787828368.770507:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787828368.770516:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb992314b40 x1874672572529536/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1787828368.770541:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787828368.770582:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828368/real 1787828368] req@ffff8cb992316580 x1874672572529280/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828393 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828368.770597:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb992316580 x1874672572529280/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828393 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828368.770605:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828368.770608:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787828368.770611:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb992316580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572529280:192.168.203.129@tcp:250:kworker.0 00000100:00100000:1.0:1787828368.770630:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb992314780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572529408:192.168.203.129@tcp:38:kworker.0 00000800:00000100:1.0:1787828368.770647:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787828368.770657:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828368/real 1787828368] req@ffff8cb992314780 x1874672572529408/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828393 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828368.770682:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb992314780 x1874672572529408/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828393 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828368.770693:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828368.770698:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787828368.770702:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb992314780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572529408:192.168.203.129@tcp:38:kworker.0 00000100:00100000:1.0:1787828368.770710:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb992314b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572529536:192.168.203.129@tcp:38:kworker.0 00000800:00000100:1.0:1787828368.770764:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787828368.770775:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828368/real 1787828368] req@ffff8cb992314b40 x1874672572529536/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828393 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828368.770785:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb992314b40 x1874672572529536/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828393 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828368.770792:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828368.770794:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787828368.770797:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb992314b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572529536:192.168.203.129@tcp:38:kworker.0 00010000:00010000:1.0:1787828369.602484:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787828370.626487:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787828371.650424:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787828372.674493:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787828373.698461:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787828373.890364:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828373.890373:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787828373.890377:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828373.890386:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 1326 00000100:00080000:1.0:1787828373.890395:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828373.890400:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:1.0:1787828373.890433:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f392c0 x1874672572529664/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828373.890460:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828373.890463:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787828373.890465:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828373.890469:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 1326 00000100:00080000:1.0:1787828373.890473:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828373.890493:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:1.0:1787828373.890505:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787828373.890509:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f392c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572529664:192.168.203.129@tcp:250:kworker.0 00000100:00100000:1.0:1787828373.890525:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f3b0c0 x1874672572529792/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828373.890541:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828373.890544:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1787828373.890546:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828373.890550:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 1326 00000100:00080000:1.0:1787828373.890555:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828373.890559:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:1.0:1787828373.890563:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787828373.890575:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f3b480 x1874672572529920/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1787828373.890585:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:3.0:1787828373.890639:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828373/real 1787828373] req@ffff8cb999f392c0 x1874672572529664/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828398 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828373.890662:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f392c0 x1874672572529664/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828398 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828373.890674:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828373.890679:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:3.0:1787828373.890684:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f392c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572529664:192.168.203.129@tcp:250:kworker.0 00000100:00100000:3.0:1787828373.890721:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f3b0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572529792:192.168.203.129@tcp:38:kworker.0 00000800:00000100:3.0:1787828373.890752:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:3.0:1787828373.890768:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828373/real 1787828373] req@ffff8cb999f3b0c0 x1874672572529792/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828398 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828373.890783:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f3b0c0 x1874672572529792/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828398 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828373.890793:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828373.890797:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:3.0:1787828373.890802:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f3b0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572529792:192.168.203.129@tcp:38:kworker.0 00000100:00100000:3.0:1787828373.890809:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f3b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572529920:192.168.203.129@tcp:38:kworker.0 00000800:00000100:3.0:1787828373.890831:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:3.0:1787828373.890848:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828373/real 1787828373] req@ffff8cb999f3b480 x1874672572529920/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828398 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828373.890860:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f3b480 x1874672572529920/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828398 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828373.890869:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828373.890873:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:3.0:1787828373.890877:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f3b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572529920:192.168.203.129@tcp:38:kworker.0 00010000:00010000:1.0:1787828374.722486:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787828375.746479:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787828376.770553:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787828377.794532:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787828378.818477:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787828379.010349:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828379.010360:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787828379.010364:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828379.010376:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 1331 00000100:00080000:0.0:1787828379.010388:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828379.010395:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:0.0:1787828379.010449:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c68f00 x1874672572530048/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828379.010485:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828379.010489:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787828379.010491:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828379.010494:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 1331 00000100:00080000:0.0:1787828379.010497:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828379.010501:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:0.0:1787828379.010512:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787828379.010525:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c69e00 x1874672572530176/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828379.010537:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1787828379.010538:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c68f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572530048:192.168.203.129@tcp:250:kworker.0 00000100:00080000:0.0:1787828379.010540:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1787828379.010544:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828379.010550:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 1331 00000100:00080000:0.0:1787828379.010554:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828379.010556:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:0.0:1787828379.010573:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787828379.010584:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c6bc00 x1874672572530304/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1787828379.010712:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c69e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572530176:192.168.203.129@tcp:38:kworker.0 00000100:00100000:3.0:1787828379.010758:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c6bc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572530304:192.168.203.129@tcp:38:kworker.0 00010000:00010000:0.0:1787828379.842534:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787828380.866445:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787828381.890632:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:2.0:1787828382.082748:0:2384:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.129:988 00000400:00000100:2.0:1787828382.082816:0:2384:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.129@tcp at host 192.168.203.129 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:2.0:1787828382.082839:0:2384:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.129@tcp 00000800:00000100:2.0:1787828382.082883:0:2384:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.129@tcp 00000800:00000100:2.0:1787828382.082921:0:2384:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.129@tcp 00000100:00100000:3.0:1787828382.083024:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828379/real 1787828382] req@ffff8cb999c69e00 x1874672572530176/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828404 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828382.083082:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c69e00 x1874672572530176/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828404 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828382.083101:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828382.083111:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:3.0:1787828382.083117:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c69e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572530176:192.168.203.129@tcp:38:kworker.0 00000100:00100000:3.0:1787828382.083131:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828379/real 1787828382] req@ffff8cb999c6bc00 x1874672572530304/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828404 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828382.083152:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c6bc00 x1874672572530304/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828404 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828382.083165:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828382.083173:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:3.0:1787828382.083179:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c6bc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572530304:192.168.203.129@tcp:38:kworker.0 00000100:00100000:3.0:1787828382.083190:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828379/real 1787828382] req@ffff8cb999c68f00 x1874672572530048/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828404 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828382.083212:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c68f00 x1874672572530048/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828404 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828382.083224:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828382.083229:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:3.0:1787828382.083325:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c68f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572530048:192.168.203.129@tcp:250:kworker.0 00010000:00010000:2.0:1787828382.914552:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787828383.938919:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787828384.130543:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828384.130557:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787828384.130562:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828384.130577:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 1336 00000100:00080000:0.0:1787828384.130590:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828384.130602:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:0.0:1787828384.130669:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c69a40 x1874672572530432/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828384.130707:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828384.130713:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787828384.130716:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828384.130722:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 1336 00000100:00080000:0.0:1787828384.130726:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828384.130731:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:0.0:1787828384.130745:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787828384.130762:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c69a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572530432:192.168.203.129@tcp:250:kworker.0 00000100:00100000:0.0:1787828384.130766:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c6b840 x1874672572530560/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828384.130809:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828384.130813:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1787828384.130829:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828384.130840:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 1336 00000100:00080000:0.0:1787828384.130848:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828384.130856:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:0.0:1787828384.130865:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000800:00000100:1.0:1787828384.130961:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787828384.130964:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c68f00 x1874672572530688/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787828384.131080:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828384/real 1787828384] req@ffff8cb999c69a40 x1874672572530432/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828409 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828384.131124:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c69a40 x1874672572530432/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828409 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828384.131173:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828384.131183:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787828384.131191:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c69a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572530432:192.168.203.129@tcp:250:kworker.0 00000100:00100000:1.0:1787828384.131245:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c6b840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572530560:192.168.203.129@tcp:38:kworker.0 00000800:00000100:1.0:1787828384.131370:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787828384.131401:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828384/real 1787828384] req@ffff8cb999c6b840 x1874672572530560/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828409 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828384.131429:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c6b840 x1874672572530560/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828409 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828384.131446:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828384.131453:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787828384.131460:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c6b840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572530560:192.168.203.129@tcp:38:kworker.0 00000100:00100000:1.0:1787828384.131475:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c68f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572530688:192.168.203.129@tcp:38:kworker.0 00000800:00000100:1.0:1787828384.131519:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787828384.131545:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828384/real 1787828384] req@ffff8cb999c68f00 x1874672572530688/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828409 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828384.131570:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c68f00 x1874672572530688/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828409 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828384.131599:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828384.131606:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787828384.131613:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c68f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572530688:192.168.203.129@tcp:38:kworker.0 00010000:00010000:1.0:1787828384.962597:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787828385.986482:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787828387.011851:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787828388.034605:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787828389.058513:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787828389.250360:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828389.250371:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787828389.250374:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828389.250383:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 1341 00000100:00080000:2.0:1787828389.250394:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828389.250401:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:2.0:1787828389.250436:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8e580 x1874672572530816/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828389.250463:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828389.250467:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787828389.250469:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828389.250474:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 1341 00000100:00080000:2.0:1787828389.250478:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828389.250496:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:1.0:1787828389.250500:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8e580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572530816:192.168.203.129@tcp:250:kworker.0 00000008:00100000:2.0:1787828389.250508:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787828389.250523:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8d680 x1874672572530944/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828389.250543:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828389.250546:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1787828389.250548:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828389.250550:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 1341 00000100:00080000:2.0:1787828389.250554:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828389.250557:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:2.0:1787828389.250560:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000800:00000100:1.0:1787828389.250566:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787828389.250579:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8c000 x1874672572531072/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787828389.250621:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828389/real 1787828389] req@ffff8cb998a8e580 x1874672572530816/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828414 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828389.250636:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8e580 x1874672572530816/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828414 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828389.250644:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828389.250647:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787828389.250650:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8e580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572530816:192.168.203.129@tcp:250:kworker.0 00000100:00100000:1.0:1787828389.250676:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8d680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572530944:192.168.203.129@tcp:38:kworker.0 00000800:00000100:1.0:1787828389.250690:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787828389.250698:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828389/real 1787828389] req@ffff8cb998a8d680 x1874672572530944/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828414 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828389.250706:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8d680 x1874672572530944/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828414 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828389.250712:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828389.250714:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787828389.250717:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8d680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572530944:192.168.203.129@tcp:38:kworker.0 00000100:00100000:1.0:1787828389.250720:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572531072:192.168.203.129@tcp:38:kworker.0 00000800:00000100:1.0:1787828389.250742:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787828389.250750:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828389/real 1787828389] req@ffff8cb998a8c000 x1874672572531072/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828414 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828389.250757:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8c000 x1874672572531072/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828414 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828389.250763:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828389.250764:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787828389.250766:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572531072:192.168.203.129@tcp:38:kworker.0 00010000:00010000:1.0:1787828390.082607:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787828391.106461:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787828392.130514:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787828393.154411:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787828394.178480:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787828394.370487:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828394.370500:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787828394.370505:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828394.370518:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 1346 00000100:00080000:1.0:1787828394.370531:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828394.370541:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:1.0:1787828394.370590:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f38f00 x1874672572531200/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828394.370630:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828394.370635:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787828394.370638:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828394.370642:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 1346 00000100:00080000:1.0:1787828394.370647:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828394.370674:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:1.0:1787828394.370688:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787828394.370706:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f38b40 x1874672572531328/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828394.370725:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1787828394.370725:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f38f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572531200:192.168.203.129@tcp:250:kworker.0 00000100:00080000:1.0:1787828394.370740:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1787828394.370745:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828394.370751:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 1346 00000100:00080000:1.0:1787828394.370756:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828394.370761:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:1.0:1787828394.370767:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787828394.370784:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f39e00 x1874672572531456/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1787828394.370849:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:3.0:1787828394.370948:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828394/real 1787828394] req@ffff8cb999f38f00 x1874672572531200/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828419 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828394.370986:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f38f00 x1874672572531200/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828419 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828394.371025:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828394.371033:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:3.0:1787828394.371040:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f38f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572531200:192.168.203.129@tcp:250:kworker.0 00000100:00100000:3.0:1787828394.371080:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f38b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572531328:192.168.203.129@tcp:38:kworker.0 00000800:00000100:3.0:1787828394.371125:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:3.0:1787828394.371145:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828394/real 1787828394] req@ffff8cb999f38b40 x1874672572531328/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828419 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828394.371162:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f38b40 x1874672572531328/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828419 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828394.371172:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828394.371176:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:3.0:1787828394.371180:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f38b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572531328:192.168.203.129@tcp:38:kworker.0 00000100:00100000:3.0:1787828394.371188:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f39e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572531456:192.168.203.129@tcp:38:kworker.0 00000800:00000100:3.0:1787828394.371211:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:3.0:1787828394.371239:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828394/real 1787828394] req@ffff8cb999f39e00 x1874672572531456/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828419 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828394.371254:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f39e00 x1874672572531456/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828419 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828394.371265:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828394.371270:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:3.0:1787828394.371277:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f39e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572531456:192.168.203.129@tcp:38:kworker.0 00010000:00010000:0.0:1787828395.202405:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787828396.226390:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787828397.250376:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787828398.274451:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787828399.298404:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787828399.490413:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828399.490425:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787828399.490430:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828399.490441:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 1351 00000100:00080000:0.0:1787828399.490454:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828399.490461:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:0.0:1787828399.490510:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c6bc00 x1874672572531584/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828399.490549:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828399.490555:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787828399.490557:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828399.490562:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 1351 00000100:00080000:0.0:1787828399.490574:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828399.490600:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:3.0:1787828399.490607:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c6bc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572531584:192.168.203.129@tcp:250:kworker.0 00000008:00100000:0.0:1787828399.490613:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787828399.490630:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c69e00 x1874672572531712/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828399.490641:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828399.490644:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1787828399.490647:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828399.490650:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 1351 00000100:00080000:0.0:1787828399.490655:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828399.490657:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:0.0:1787828399.490662:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787828399.490671:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c6b480 x1874672572531840/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1787828399.490692:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:3.0:1787828399.490741:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828399/real 1787828399] req@ffff8cb999c6bc00 x1874672572531584/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828424 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828399.490757:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c6bc00 x1874672572531584/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828424 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828399.490767:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828399.490771:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:3.0:1787828399.490775:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c6bc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572531584:192.168.203.129@tcp:250:kworker.0 00000100:00100000:3.0:1787828399.490819:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c69e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572531712:192.168.203.129@tcp:38:kworker.0 00000800:00000100:3.0:1787828399.490841:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:3.0:1787828399.490852:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828399/real 1787828399] req@ffff8cb999c69e00 x1874672572531712/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828424 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828399.490861:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c69e00 x1874672572531712/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828424 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828399.490881:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828399.490884:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:3.0:1787828399.490887:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c69e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572531712:192.168.203.129@tcp:38:kworker.0 00000100:00100000:3.0:1787828399.490892:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c6b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572531840:192.168.203.129@tcp:38:kworker.0 00000800:00000100:3.0:1787828399.490909:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:3.0:1787828399.490918:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828399/real 1787828399] req@ffff8cb999c6b480 x1874672572531840/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828424 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828399.490943:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c6b480 x1874672572531840/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828424 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828399.490950:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828399.490953:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:3.0:1787828399.490956:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c6b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572531840:192.168.203.129@tcp:38:kworker.0 00010000:00010000:1.0:1787828400.322452:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787828401.346564:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787828402.370441:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787828403.394566:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787828404.418503:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787828404.610398:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828404.610435:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787828404.610441:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828404.610455:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 1356 00000100:00080000:0.0:1787828404.610481:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828404.610493:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:0.0:1787828404.610543:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c69680 x1874672572531968/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828404.610572:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828404.610589:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787828404.610621:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828404.610627:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 1356 00000100:00080000:0.0:1787828404.610634:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828404.610640:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:0.0:1787828404.610655:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787828404.610687:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c68000 x1874672572532096/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828404.610697:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828404.610701:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1787828404.610703:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828404.610707:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 1356 00000100:00080000:0.0:1787828404.610710:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828404.610713:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:0.0:1787828404.610717:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787828404.610723:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c69680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572531968:192.168.203.129@tcp:250:kworker.0 00000100:00100000:0.0:1787828404.610729:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c692c0 x1874672572532224/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1787828404.610885:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:3.0:1787828404.610966:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828404/real 1787828404] req@ffff8cb999c69680 x1874672572531968/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828429 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828404.611000:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c69680 x1874672572531968/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828429 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828404.611014:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828404.611020:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:3.0:1787828404.611037:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c69680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572531968:192.168.203.129@tcp:250:kworker.0 00000100:00100000:3.0:1787828404.611047:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c68000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572532096:192.168.203.129@tcp:38:kworker.0 00000800:00000100:3.0:1787828404.611079:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:3.0:1787828404.611094:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828404/real 1787828404] req@ffff8cb999c68000 x1874672572532096/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828429 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828404.611121:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c68000 x1874672572532096/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828429 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828404.611128:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828404.611131:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:3.0:1787828404.611134:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c68000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572532096:192.168.203.129@tcp:38:kworker.0 00000100:00100000:3.0:1787828404.611160:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c692c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572532224:192.168.203.129@tcp:38:kworker.0 00000800:00000100:3.0:1787828404.611201:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:3.0:1787828404.611219:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828404/real 1787828404] req@ffff8cb999c692c0 x1874672572532224/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828429 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828404.611238:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c692c0 x1874672572532224/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828429 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828404.611252:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828404.611257:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:3.0:1787828404.611264:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c692c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572532224:192.168.203.129@tcp:38:kworker.0 00010000:00010000:1.0:1787828405.442559:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787828406.466469:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787828407.490460:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787828408.514528:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787828409.538502:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787828409.730357:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828409.730368:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787828409.730373:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828409.730385:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 1361 00000100:00080000:0.0:1787828409.730396:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828409.730405:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:0.0:1787828409.730459:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c6a580 x1874672572532352/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828409.730515:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828409.730519:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787828409.730521:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828409.730525:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 1361 00000100:00080000:0.0:1787828409.730530:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828409.730535:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:0.0:1787828409.730547:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787828409.730560:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c6a1c0 x1874672572532480/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1787828409.730593:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c6a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572532352:192.168.203.129@tcp:250:kworker.0 00000100:00080000:0.0:1787828409.730594:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828409.730606:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1787828409.730610:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828409.730616:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 1361 00000100:00080000:0.0:1787828409.730620:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828409.730625:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:0.0:1787828409.730631:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787828409.730647:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c692c0 x1874672572532608/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1787828409.730684:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:3.0:1787828409.730781:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828409/real 1787828409] req@ffff8cb999c6a580 x1874672572532352/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828434 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828409.730801:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c6a580 x1874672572532352/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828434 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828409.730813:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828409.730820:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:3.0:1787828409.730826:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c6a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572532352:192.168.203.129@tcp:250:kworker.0 00000100:00100000:3.0:1787828409.730858:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c6a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572532480:192.168.203.129@tcp:38:kworker.0 00000800:00000100:3.0:1787828409.730896:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:3.0:1787828409.730918:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828409/real 1787828409] req@ffff8cb999c6a1c0 x1874672572532480/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828434 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828409.730938:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c6a1c0 x1874672572532480/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828434 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828409.730953:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828409.730959:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:3.0:1787828409.730965:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c6a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572532480:192.168.203.129@tcp:38:kworker.0 00000100:00100000:3.0:1787828409.730974:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c692c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572532608:192.168.203.129@tcp:38:kworker.0 00000800:00000100:3.0:1787828409.730998:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:3.0:1787828409.731011:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828409/real 1787828409] req@ffff8cb999c692c0 x1874672572532608/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828434 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828409.731023:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c692c0 x1874672572532608/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828434 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828409.731046:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828409.731050:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:3.0:1787828409.731053:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c692c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572532608:192.168.203.129@tcp:38:kworker.0 00010000:00010000:0.0:1787828410.562535:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787828411.586437:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787828412.610461:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787828413.634441:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787828414.658555:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787828414.850426:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828414.850439:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787828414.850445:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828414.850458:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 1367 00000100:00080000:3.0:1787828414.850473:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828414.850483:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:3.0:1787828414.850531:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb992317480 x1874672572532736/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828414.850570:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828414.850575:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787828414.850578:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828414.850583:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 1367 00000100:00080000:3.0:1787828414.850587:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828414.850592:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:3.0:1787828414.850603:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787828414.850619:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb992314f00 x1874672572532864/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828414.850634:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828414.850638:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1787828414.850658:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1787828414.850670:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb992317480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572532736:192.168.203.129@tcp:250:kworker.0 00000100:00080000:3.0:1787828414.850671:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 1367 00000100:00080000:3.0:1787828414.850701:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828414.850713:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:3.0:1787828414.850722:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787828414.850744:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb992315e00 x1874672572532992/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1787828414.850822:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787828414.850907:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828414/real 1787828414] req@ffff8cb992317480 x1874672572532736/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828439 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828414.850937:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb992317480 x1874672572532736/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828439 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828414.850958:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828414.850965:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787828414.850974:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb992317480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572532736:192.168.203.129@tcp:250:kworker.0 00000100:00100000:1.0:1787828414.850989:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb992314f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572532864:192.168.203.129@tcp:38:kworker.0 00000800:00000100:1.0:1787828414.851038:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787828414.851061:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828414/real 1787828414] req@ffff8cb992314f00 x1874672572532864/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828439 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828414.851092:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb992314f00 x1874672572532864/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828439 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828414.851104:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828414.851109:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787828414.851113:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb992314f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572532864:192.168.203.129@tcp:38:kworker.0 00000100:00100000:1.0:1787828414.851161:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb992315e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572532992:192.168.203.129@tcp:38:kworker.0 00000800:00000100:1.0:1787828414.851205:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787828414.851229:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828414/real 1787828414] req@ffff8cb992315e00 x1874672572532992/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828439 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828414.851309:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb992315e00 x1874672572532992/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828439 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828414.851321:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828414.851326:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787828414.851330:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb992315e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572532992:192.168.203.129@tcp:38:kworker.0 00010000:00010000:1.0:1787828415.682394:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787828416.706503:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787828417.730384:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787828418.754489:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787828419.778463:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787828419.970496:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828419.970506:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787828419.970511:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828419.970523:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 1372 00000100:00080000:1.0:1787828419.970535:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828419.970555:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:1.0:1787828419.970625:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f39a40 x1874672572533120/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828419.970704:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828419.970709:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787828419.970712:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828419.970718:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 1372 00000100:00080000:1.0:1787828419.970724:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828419.970730:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:1.0:1787828419.970743:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787828419.970750:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f39a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572533120:192.168.203.129@tcp:250:kworker.0 00000100:00100000:1.0:1787828419.970786:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f3b840 x1874672572533248/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828419.970803:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828419.970808:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1787828419.970810:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828419.970818:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 1372 00000100:00080000:1.0:1787828419.970825:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828419.970829:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:1.0:1787828419.970837:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000800:00000100:3.0:1787828419.970854:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787828419.970857:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f39e00 x1874672572533376/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1787828419.971042:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828419/real 1787828419] req@ffff8cb999f39a40 x1874672572533120/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828444 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828419.971223:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f39a40 x1874672572533120/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828444 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828419.971237:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828419.971243:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:3.0:1787828419.971360:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f39a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572533120:192.168.203.129@tcp:250:kworker.0 00000100:00100000:3.0:1787828419.971405:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f3b840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572533248:192.168.203.129@tcp:38:kworker.0 00000800:00000100:3.0:1787828419.971444:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:3.0:1787828419.971461:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828419/real 1787828419] req@ffff8cb999f3b840 x1874672572533248/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828444 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828419.971476:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f3b840 x1874672572533248/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828444 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828419.971512:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828419.971516:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:3.0:1787828419.971520:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f3b840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572533248:192.168.203.129@tcp:38:kworker.0 00000100:00100000:3.0:1787828419.971528:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f39e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572533376:192.168.203.129@tcp:38:kworker.0 00000800:00000100:3.0:1787828419.971550:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:3.0:1787828419.971573:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828419/real 1787828419] req@ffff8cb999f39e00 x1874672572533376/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828444 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828419.971585:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f39e00 x1874672572533376/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828444 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828419.971594:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828419.971598:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:3.0:1787828419.971601:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f39e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572533376:192.168.203.129@tcp:38:kworker.0 00010000:00010000:1.0:1787828420.802429:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787828421.826382:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787828422.850436:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787828423.874579:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787828424.898431:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787828425.090382:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828425.090390:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787828425.090394:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828425.090403:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 1377 00000100:00080000:1.0:1787828425.090412:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828425.090418:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:1.0:1787828425.090468:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f38b40 x1874672572533504/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828425.090496:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828425.090500:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787828425.090533:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828425.090537:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 1377 00000100:00080000:1.0:1787828425.090541:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828425.090545:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:1.0:1787828425.090555:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787828425.090566:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f38b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572533504:192.168.203.129@tcp:250:kworker.0 00000100:00100000:1.0:1787828425.090568:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f38f00 x1874672572533632/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828425.090582:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828425.090585:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1787828425.090587:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828425.090590:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 1377 00000100:00080000:1.0:1787828425.090593:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828425.090595:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:1.0:1787828425.090599:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787828425.090624:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f3b480 x1874672572533760/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1787828425.090647:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:3.0:1787828425.090696:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828425/real 1787828425] req@ffff8cb999f38b40 x1874672572533504/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828450 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828425.090719:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f38b40 x1874672572533504/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828450 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828425.090732:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828425.090738:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:3.0:1787828425.090751:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f38b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572533504:192.168.203.129@tcp:250:kworker.0 00000100:00100000:3.0:1787828425.090774:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f38f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572533632:192.168.203.129@tcp:38:kworker.0 00000800:00000100:3.0:1787828425.090795:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:3.0:1787828425.090805:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828425/real 1787828425] req@ffff8cb999f38f00 x1874672572533632/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828450 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828425.090816:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f38f00 x1874672572533632/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828450 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828425.090823:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828425.090825:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:3.0:1787828425.090828:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f38f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572533632:192.168.203.129@tcp:38:kworker.0 00000100:00100000:3.0:1787828425.090833:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f3b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572533760:192.168.203.129@tcp:38:kworker.0 00000800:00000100:3.0:1787828425.090846:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:3.0:1787828425.090852:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828425/real 1787828425] req@ffff8cb999f3b480 x1874672572533760/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828450 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828425.090861:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f3b480 x1874672572533760/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828450 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828425.090867:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828425.090870:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:3.0:1787828425.090872:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f3b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572533760:192.168.203.129@tcp:38:kworker.0 00010000:00010000:0.0:1787828425.922416:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787828426.946703:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787828427.970495:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787828428.994476:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787828430.018494:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787828430.210354:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828430.210362:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787828430.210366:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828430.210374:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 1382 00000100:00080000:2.0:1787828430.210383:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828430.210389:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:2.0:1787828430.210420:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8f480 x1874672572533888/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828430.210443:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828430.210446:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787828430.210461:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828430.210465:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 1382 00000100:00080000:2.0:1787828430.210469:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828430.210472:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:2.0:1787828430.210481:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787828430.210486:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572533888:192.168.203.129@tcp:250:kworker.0 00000100:00100000:2.0:1787828430.210493:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8cb40 x1874672572534016/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828430.210503:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828430.210506:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1787828430.210507:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828430.210510:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 1382 00000100:00080000:2.0:1787828430.210514:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828430.210516:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:2.0:1787828430.210519:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787828430.210527:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8e940 x1874672572534144/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1787828430.210542:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:3.0:1787828430.210612:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828430/real 1787828430] req@ffff8cb998a8f480 x1874672572533888/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828455 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828430.210631:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8f480 x1874672572533888/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828455 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828430.210640:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828430.210644:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:3.0:1787828430.210647:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572533888:192.168.203.129@tcp:250:kworker.0 00000100:00100000:3.0:1787828430.210667:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8cb40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572534016:192.168.203.129@tcp:38:kworker.0 00000800:00000100:3.0:1787828430.210684:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:3.0:1787828430.210693:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828430/real 1787828430] req@ffff8cb998a8cb40 x1874672572534016/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828455 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828430.210702:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8cb40 x1874672572534016/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828455 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828430.210708:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828430.210711:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:3.0:1787828430.210713:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8cb40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572534016:192.168.203.129@tcp:38:kworker.0 00000100:00100000:3.0:1787828430.210717:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8e940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572534144:192.168.203.129@tcp:38:kworker.0 00000800:00000100:3.0:1787828430.210727:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:3.0:1787828430.210734:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828430/real 1787828430] req@ffff8cb998a8e940 x1874672572534144/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828455 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828430.210742:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8e940 x1874672572534144/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828455 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828430.210759:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828430.210762:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:3.0:1787828430.210765:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8e940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572534144:192.168.203.129@tcp:38:kworker.0 00010000:00010000:1.0:1787828431.042522:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787828432.066402:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787828433.090581:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787828434.114506:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787828435.138396:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787828435.330397:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828435.330406:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787828435.330412:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828435.330421:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 1387 00000100:00080000:3.0:1787828435.330431:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828435.330450:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:3.0:1787828435.330485:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb992316940 x1874672572534272/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828435.330523:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828435.330527:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787828435.330529:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828435.330546:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 1387 00000100:00080000:3.0:1787828435.330552:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828435.330556:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:3.0:1787828435.330577:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787828435.330587:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb992316940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572534272:192.168.203.129@tcp:250:kworker.0 00000100:00100000:3.0:1787828435.330593:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb9923161c0 x1874672572534400/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828435.330605:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828435.330608:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1787828435.330622:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828435.330627:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 1387 00000100:00080000:3.0:1787828435.330632:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828435.330643:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:3.0:1787828435.330649:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000800:00000100:1.0:1787828435.330661:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:3.0:1787828435.330663:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb992315a40 x1874672572534528/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787828435.330711:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828435/real 1787828435] req@ffff8cb992316940 x1874672572534272/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828460 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828435.330730:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb992316940 x1874672572534272/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828460 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828435.330764:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828435.330771:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787828435.330775:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb992316940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572534272:192.168.203.129@tcp:250:kworker.0 00000100:00100000:1.0:1787828435.330801:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb9923161c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572534400:192.168.203.129@tcp:38:kworker.0 00000800:00000100:1.0:1787828435.330827:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787828435.330841:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828435/real 1787828435] req@ffff8cb9923161c0 x1874672572534400/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828460 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828435.330888:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb9923161c0 x1874672572534400/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828460 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828435.330896:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828435.330899:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787828435.330902:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb9923161c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572534400:192.168.203.129@tcp:38:kworker.0 00000100:00100000:1.0:1787828435.330907:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb992315a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572534528:192.168.203.129@tcp:38:kworker.0 00000800:00000100:1.0:1787828435.330933:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787828435.330942:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828435/real 1787828435] req@ffff8cb992315a40 x1874672572534528/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828460 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828435.330951:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb992315a40 x1874672572534528/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828460 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828435.330957:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828435.330960:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787828435.330962:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb992315a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572534528:192.168.203.129@tcp:38:kworker.0 00010000:00010000:0.0:1787828436.162559:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787828437.186517:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787828438.210463:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787828439.234481:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787828440.258465:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787828440.450400:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828440.450416:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787828440.450423:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828440.450439:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 1392 00000100:00080000:3.0:1787828440.450456:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828440.450467:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:3.0:1787828440.450553:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb9923170c0 x1874672572534656/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828440.450600:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828440.450607:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787828440.450611:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828440.450619:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 1392 00000100:00080000:3.0:1787828440.450628:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828440.450637:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:3.0:1787828440.450655:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787828440.450662:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb9923170c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572534656:192.168.203.129@tcp:250:kworker.0 00000100:00100000:3.0:1787828440.450686:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb992315a40 x1874672572534784/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828440.450746:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828440.450772:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1787828440.450775:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:1.0:1787828440.450783:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00080000:3.0:1787828440.450787:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 1392 00000100:00080000:3.0:1787828440.450798:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828440.450805:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:3.0:1787828440.450815:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787828440.450861:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb9923161c0 x1874672572534912/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787828440.450864:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828440/real 1787828440] req@ffff8cb9923170c0 x1874672572534656/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828465 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828440.450916:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb9923170c0 x1874672572534656/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828465 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828440.450959:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828440.450966:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787828440.450972:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb9923170c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572534656:192.168.203.129@tcp:250:kworker.0 00000100:00100000:1.0:1787828440.451026:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb992315a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572534784:192.168.203.129@tcp:38:kworker.0 00000800:00000100:1.0:1787828440.451052:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787828440.451065:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828440/real 1787828440] req@ffff8cb992315a40 x1874672572534784/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828465 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828440.451098:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb992315a40 x1874672572534784/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828465 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828440.451110:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828440.451114:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787828440.451119:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb992315a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572534784:192.168.203.129@tcp:38:kworker.0 00000100:00100000:1.0:1787828440.451126:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb9923161c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572534912:192.168.203.129@tcp:38:kworker.0 00000800:00000100:1.0:1787828440.451170:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787828440.451189:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828440/real 1787828440] req@ffff8cb9923161c0 x1874672572534912/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828465 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828440.451214:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb9923161c0 x1874672572534912/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828465 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828440.451231:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828440.451237:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787828440.451244:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb9923161c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572534912:192.168.203.129@tcp:38:kworker.0 00010000:00010000:0.0:1787828441.282570:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787828442.306558:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787828443.330463:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787828444.354431:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787828445.378544:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787828445.570382:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828445.570394:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787828445.570399:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828445.570412:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 1397 00000100:00080000:3.0:1787828445.570424:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828445.570434:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:3.0:1787828445.570514:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb992316940 x1874672572535040/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828445.570554:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828445.570559:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787828445.570562:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828445.570567:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 1397 00000100:00080000:3.0:1787828445.570572:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828445.570577:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:3.0:1787828445.570605:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787828445.570624:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb992315e00 x1874672572535168/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828445.570639:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828445.570644:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1787828445.570647:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828445.570652:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 1397 00000100:00080000:3.0:1787828445.570658:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828445.570668:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:1.0:1787828445.570676:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb992316940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572535040:192.168.203.129@tcp:250:kworker.0 00000008:00100000:3.0:1787828445.570677:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787828445.570695:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb992314f00 x1874672572535296/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787828445.570846:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb992315e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572535168:192.168.203.129@tcp:38:kworker.0 00000100:00100000:1.0:1787828445.570925:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb992314f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572535296:192.168.203.129@tcp:38:kworker.0 00010000:00010000:1.0:1787828446.402486:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787828447.426384:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787828448.450987:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:3.0:1787828448.644058:0:2381:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.129:988 00000400:00000100:3.0:1787828448.644122:0:2381:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.129@tcp at host 192.168.203.129 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:3.0:1787828448.644141:0:2381:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.129@tcp 00000800:00000100:3.0:1787828448.644192:0:2381:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.129@tcp 00000800:00000100:3.0:1787828448.644205:0:2381:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.129@tcp 00000100:00100000:0.0:1787828448.644250:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828445/real 1787828448] req@ffff8cb992314f00 x1874672572535296/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828470 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828448.644321:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb992314f00 x1874672572535296/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828470 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828448.644337:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828448.644345:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787828448.644352:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb992314f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572535296:192.168.203.129@tcp:38:kworker.0 00000100:00100000:0.0:1787828448.644374:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828445/real 1787828448] req@ffff8cb992316940 x1874672572535040/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828470 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828448.644392:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb992316940 x1874672572535040/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828470 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828448.644415:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828448.644418:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787828448.644422:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb992316940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572535040:192.168.203.129@tcp:250:kworker.0 00000100:00100000:0.0:1787828448.644435:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828445/real 1787828448] req@ffff8cb992315e00 x1874672572535168/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828470 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828448.644464:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb992315e00 x1874672572535168/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828470 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828448.644471:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828448.644474:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787828448.644477:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb992315e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572535168:192.168.203.129@tcp:38:kworker.0 00010000:00010000:1.0:1787828449.474450:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787828450.499134:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787828450.690509:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828450.690525:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787828450.690530:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828450.690543:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 1402 00000100:00080000:1.0:1787828450.690556:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828450.690566:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:1.0:1787828450.690610:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f3b0c0 x1874672572535424/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828450.690685:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828450.690690:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787828450.690692:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828450.690697:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 1402 00000100:00080000:1.0:1787828450.690702:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828450.690706:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:1.0:1787828450.690717:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787828450.690739:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f392c0 x1874672572535552/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828450.690766:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828450.690773:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1787828450.690777:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1787828450.690786:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f3b0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572535424:192.168.203.129@tcp:250:kworker.0 00000100:00080000:1.0:1787828450.690789:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 1402 00000100:00080000:1.0:1787828450.690802:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828450.690812:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:1.0:1787828450.690821:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787828450.690844:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f3a580 x1874672572535680/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1787828450.690892:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787828450.690964:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828450/real 1787828450] req@ffff8cb999f3b0c0 x1874672572535424/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828475 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828450.690997:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f3b0c0 x1874672572535424/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828475 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828450.691016:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828450.691024:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787828450.691030:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f3b0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572535424:192.168.203.129@tcp:250:kworker.0 00000100:00100000:0.0:1787828450.691042:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f392c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572535552:192.168.203.129@tcp:38:kworker.0 00000800:00000100:0.0:1787828450.691100:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787828450.691122:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828450/real 1787828450] req@ffff8cb999f392c0 x1874672572535552/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828475 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828450.691144:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f392c0 x1874672572535552/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828475 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828450.691159:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828450.691165:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787828450.691171:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f392c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572535552:192.168.203.129@tcp:38:kworker.0 00000100:00100000:0.0:1787828450.691209:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f3a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572535680:192.168.203.129@tcp:38:kworker.0 00000800:00000100:0.0:1787828450.691245:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787828450.691310:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828450/real 1787828450] req@ffff8cb999f3a580 x1874672572535680/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828475 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828450.691336:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f3a580 x1874672572535680/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828475 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828450.691350:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828450.691356:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787828450.691362:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f3a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572535680:192.168.203.129@tcp:38:kworker.0 00010000:00010000:0.0:1787828451.522976:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787828452.546568:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787828453.570482:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787828454.594839:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787828455.618593:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787828455.810479:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828455.810491:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787828455.810497:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828455.810512:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 1407 00000100:00080000:3.0:1787828455.810540:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828455.810551:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:3.0:1787828455.810598:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb992317480 x1874672572535808/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828455.810669:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828455.810675:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787828455.810679:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828455.810684:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 1407 00000100:00080000:3.0:1787828455.810690:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828455.810698:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:3.0:1787828455.810723:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787828455.810759:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb992317480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572535808:192.168.203.129@tcp:250:kworker.0 00000100:00100000:3.0:1787828455.810759:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb992314b40 x1874672572535936/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828455.810791:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828455.810796:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1787828455.810799:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828455.810808:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 1407 00000100:00080000:3.0:1787828455.810818:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828455.810828:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:3.0:1787828455.810836:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787828455.810869:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb992314780 x1874672572536064/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1787828455.810887:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787828455.810997:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828455/real 1787828455] req@ffff8cb992317480 x1874672572535808/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828480 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828455.811034:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb992317480 x1874672572535808/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828480 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828455.811050:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828455.811055:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787828455.811061:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb992317480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572535808:192.168.203.129@tcp:250:kworker.0 00000100:00100000:1.0:1787828455.811106:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb992314b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572535936:192.168.203.129@tcp:38:kworker.0 00000800:00000100:1.0:1787828455.811154:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787828455.811181:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828455/real 1787828455] req@ffff8cb992314b40 x1874672572535936/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828480 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828455.811213:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb992314b40 x1874672572535936/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828480 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828455.811232:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828455.811241:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787828455.811250:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb992314b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572535936:192.168.203.129@tcp:38:kworker.0 00000100:00100000:1.0:1787828455.811308:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb992314780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572536064:192.168.203.129@tcp:38:kworker.0 00000800:00000100:1.0:1787828455.811358:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787828455.811379:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828455/real 1787828455] req@ffff8cb992314780 x1874672572536064/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828480 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828455.811432:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb992314780 x1874672572536064/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828480 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828455.811468:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828455.811473:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787828455.811478:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb992314780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572536064:192.168.203.129@tcp:38:kworker.0 00010000:00010000:1.0:1787828456.642773:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787828457.666696:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787828458.690526:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787828459.714536:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787828460.738509:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787828460.930396:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828460.930431:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787828460.930434:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828460.930443:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 1413 00000100:00080000:0.0:1787828460.930451:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828460.930457:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:0.0:1787828460.930497:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c68000 x1874672572536192/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828460.930524:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828460.930527:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787828460.930530:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828460.930533:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 1413 00000100:00080000:0.0:1787828460.930537:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828460.930542:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:0.0:1787828460.930553:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787828460.930568:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c68000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572536192:192.168.203.129@tcp:250:kworker.0 00000100:00100000:0.0:1787828460.930569:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c69680 x1874672572536320/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828460.930583:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828460.930586:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1787828460.930587:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828460.930590:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 1413 00000100:00080000:0.0:1787828460.930594:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828460.930596:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:0.0:1787828460.930600:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787828460.930610:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c6b480 x1874672572536448/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1787828460.930630:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787828460.930665:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828460/real 1787828460] req@ffff8cb999c68000 x1874672572536192/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828485 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828460.930678:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c68000 x1874672572536192/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828485 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828460.930686:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828460.930688:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787828460.930691:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c68000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572536192:192.168.203.129@tcp:250:kworker.0 00000100:00100000:1.0:1787828460.930711:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c69680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572536320:192.168.203.129@tcp:38:kworker.0 00000800:00000100:1.0:1787828460.930725:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787828460.930733:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828460/real 1787828460] req@ffff8cb999c69680 x1874672572536320/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828485 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828460.930741:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c69680 x1874672572536320/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828485 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828460.930757:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828460.930759:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787828460.930762:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c69680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572536320:192.168.203.129@tcp:38:kworker.0 00000100:00100000:1.0:1787828460.930766:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c6b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572536448:192.168.203.129@tcp:38:kworker.0 00000800:00000100:1.0:1787828460.930779:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787828460.930786:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828460/real 1787828460] req@ffff8cb999c6b480 x1874672572536448/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828485 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828460.930793:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c6b480 x1874672572536448/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828485 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828460.930798:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828460.930800:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787828460.930801:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c6b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572536448:192.168.203.129@tcp:38:kworker.0 00010000:00010000:1.0:1787828461.762442:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787828462.786484:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787828463.810426:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787828464.834499:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787828465.858489:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787828466.050422:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828466.050431:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787828466.050435:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828466.050444:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 1418 00000100:00080000:3.0:1787828466.050472:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828466.050479:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:3.0:1787828466.050511:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb992316580 x1874672572536576/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828466.050544:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828466.050547:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787828466.050549:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828466.050552:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 1418 00000100:00080000:3.0:1787828466.050555:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828466.050559:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:3.0:1787828466.050579:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787828466.050592:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb992314780 x1874672572536704/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787828466.050601:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb992316580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572536576:192.168.203.129@tcp:250:kworker.0 00000100:00080000:3.0:1787828466.050605:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828466.050611:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1787828466.050613:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828466.050617:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 1418 00000100:00080000:3.0:1787828466.050619:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828466.050622:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:3.0:1787828466.050626:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787828466.050637:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb992314b40 x1874672572536832/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1787828466.050669:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787828466.050716:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828466/real 1787828466] req@ffff8cb992316580 x1874672572536576/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828491 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828466.050735:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb992316580 x1874672572536576/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828491 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828466.050758:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828466.050762:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787828466.050765:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb992316580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572536576:192.168.203.129@tcp:250:kworker.0 00000100:00100000:1.0:1787828466.050787:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb992314780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572536704:192.168.203.129@tcp:38:kworker.0 00000800:00000100:1.0:1787828466.050808:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787828466.050820:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828466/real 1787828466] req@ffff8cb992314780 x1874672572536704/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828491 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828466.050830:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb992314780 x1874672572536704/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828491 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828466.050837:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828466.050840:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787828466.050842:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb992314780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572536704:192.168.203.129@tcp:38:kworker.0 00000100:00100000:1.0:1787828466.050847:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb992314b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572536832:192.168.203.129@tcp:38:kworker.0 00000800:00000100:1.0:1787828466.050860:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787828466.050868:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828466/real 1787828466] req@ffff8cb992314b40 x1874672572536832/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828491 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828466.050877:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb992314b40 x1874672572536832/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828491 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828466.050883:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828466.050885:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787828466.050888:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb992314b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572536832:192.168.203.129@tcp:38:kworker.0 00010000:00010000:0.0:1787828466.882522:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787828467.906440:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787828468.930431:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787828469.954505:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787828470.978578:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787828471.170438:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828471.170447:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787828471.170452:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828471.170462:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 1423 00000100:00080000:1.0:1787828471.170473:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828471.170480:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:1.0:1787828471.170518:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f383c0 x1874672572536960/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828471.170553:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828471.170557:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787828471.170559:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828471.170562:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 1423 00000100:00080000:1.0:1787828471.170566:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828471.170569:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:1.0:1787828471.170579:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787828471.170593:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f3a580 x1874672572537088/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828471.170606:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1787828471.170613:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f383c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572536960:192.168.203.129@tcp:250:kworker.0 00000100:00080000:1.0:1787828471.170613:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1787828471.170617:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828471.170659:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 1423 00000100:00080000:1.0:1787828471.170710:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000800:00000100:0.0:1787828471.170722:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00080000:1.0:1787828471.170725:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:1.0:1787828471.170758:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787828471.170784:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828471/real 1787828471] req@ffff8cb999f383c0 x1874672572536960/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828496 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828471.170795:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f392c0 x1874672572537216/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787828471.170822:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f383c0 x1874672572536960/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828496 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828471.170842:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828471.170849:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787828471.170884:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f383c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572536960:192.168.203.129@tcp:250:kworker.0 00000100:00100000:0.0:1787828471.170919:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f3a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572537088:192.168.203.129@tcp:38:kworker.0 00000800:00000100:0.0:1787828471.170962:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787828471.170974:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828471/real 1787828471] req@ffff8cb999f3a580 x1874672572537088/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828496 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828471.170986:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f3a580 x1874672572537088/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828496 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828471.170995:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828471.170999:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787828471.171003:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f3a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572537088:192.168.203.129@tcp:38:kworker.0 00000100:00100000:0.0:1787828471.171021:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f392c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572537216:192.168.203.129@tcp:38:kworker.0 00000800:00000100:0.0:1787828471.171041:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787828471.171051:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828471/real 1787828471] req@ffff8cb999f392c0 x1874672572537216/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828496 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828471.171072:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f392c0 x1874672572537216/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828496 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828471.171079:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828471.171082:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787828471.171084:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f392c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572537216:192.168.203.129@tcp:38:kworker.0 00010000:00010000:1.0:1787828472.002504:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787828473.026552:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787828474.050444:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787828475.074407:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787828476.098429:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787828476.290337:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828476.290346:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787828476.290350:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828476.290359:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 1428 00000100:00080000:1.0:1787828476.290368:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828476.290374:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:1.0:1787828476.290412:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f3b0c0 x1874672572537344/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828476.290433:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828476.290435:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787828476.290437:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828476.290440:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 1428 00000100:00080000:1.0:1787828476.290443:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828476.290447:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:1.0:1787828476.290458:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787828476.290473:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f3b480 x1874672572537472/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787828476.290481:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f3b0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572537344:192.168.203.129@tcp:250:kworker.0 00000100:00080000:1.0:1787828476.290503:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828476.290505:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1787828476.290507:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828476.290511:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 1428 00000100:00080000:1.0:1787828476.290528:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828476.290534:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000800:00000100:0.0:1787828476.290538:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000008:00100000:1.0:1787828476.290544:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787828476.290562:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f38f00 x1874672572537600/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787828476.290578:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828476/real 1787828476] req@ffff8cb999f3b0c0 x1874672572537344/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828501 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828476.290594:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f3b0c0 x1874672572537344/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828501 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828476.290602:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828476.290606:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787828476.290609:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f3b0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572537344:192.168.203.129@tcp:250:kworker.0 00000100:00100000:0.0:1787828476.290627:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f3b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572537472:192.168.203.129@tcp:38:kworker.0 00000800:00000100:0.0:1787828476.290653:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787828476.290662:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828476/real 1787828476] req@ffff8cb999f3b480 x1874672572537472/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828501 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828476.290671:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f3b480 x1874672572537472/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828501 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828476.290676:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828476.290679:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787828476.290681:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f3b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572537472:192.168.203.129@tcp:38:kworker.0 00000100:00100000:0.0:1787828476.290685:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f38f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572537600:192.168.203.129@tcp:38:kworker.0 00000800:00000100:0.0:1787828476.290700:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787828476.290711:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828476/real 1787828476] req@ffff8cb999f38f00 x1874672572537600/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828501 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828476.290724:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f38f00 x1874672572537600/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828501 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828476.290734:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828476.290737:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787828476.290741:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f38f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572537600:192.168.203.129@tcp:38:kworker.0 00010000:00010000:1.0:1787828477.122465:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787828478.146430:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787828479.170431:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787828480.194465:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787828481.218469:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787828481.410365:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828481.410374:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787828481.410377:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828481.410386:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 1433 00000100:00080000:0.0:1787828481.410417:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828481.410423:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:0.0:1787828481.410453:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c69e00 x1874672572537728/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828481.410477:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828481.410479:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787828481.410481:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828481.410483:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 1433 00000100:00080000:0.0:1787828481.410486:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828481.410489:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:0.0:1787828481.410497:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787828481.410509:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c6bc00 x1874672572537856/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787828481.410509:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c69e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572537728:192.168.203.129@tcp:250:kworker.0 00000100:00080000:0.0:1787828481.410523:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828481.410526:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1787828481.410527:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828481.410530:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 1433 00000100:00080000:0.0:1787828481.410532:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828481.410534:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:0.0:1787828481.410537:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787828481.410547:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c68f00 x1874672572537984/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1787828481.410569:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787828481.410604:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828481/real 1787828481] req@ffff8cb999c69e00 x1874672572537728/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828506 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828481.410616:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c69e00 x1874672572537728/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828506 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828481.410624:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828481.410627:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787828481.410630:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c69e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572537728:192.168.203.129@tcp:250:kworker.0 00000100:00100000:1.0:1787828481.410649:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c6bc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572537856:192.168.203.129@tcp:38:kworker.0 00000800:00000100:1.0:1787828481.410671:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787828481.410699:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828481/real 1787828481] req@ffff8cb999c6bc00 x1874672572537856/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828506 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828481.410711:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c6bc00 x1874672572537856/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828506 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828481.410719:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828481.410722:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787828481.410725:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c6bc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572537856:192.168.203.129@tcp:38:kworker.0 00000100:00100000:1.0:1787828481.410731:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c68f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572537984:192.168.203.129@tcp:38:kworker.0 00000800:00000100:1.0:1787828481.410746:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787828481.410755:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828481/real 1787828481] req@ffff8cb999c68f00 x1874672572537984/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828506 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828481.410789:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c68f00 x1874672572537984/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828506 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828481.410795:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828481.410797:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787828481.410799:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c68f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572537984:192.168.203.129@tcp:38:kworker.0 00010000:00010000:0.0:1787828482.242430:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787828483.266426:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787828484.290526:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787828485.314455:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787828486.338470:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787828486.530308:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828486.530315:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787828486.530319:0:37:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828486.530327:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 1438 00000100:00080000:0.0:1787828486.530336:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828486.530343:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:0.0:1787828486.530374:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c6b840 x1874672572538112/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828486.530394:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828486.530398:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787828486.530399:0:37:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828486.530401:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 1438 00000100:00080000:0.0:1787828486.530404:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828486.530407:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:0.0:1787828486.530415:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787828486.530423:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c69a40 x1874672572538240/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828486.530431:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1787828486.530434:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c6b840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572538112:192.168.203.129@tcp:250:kworker.0 00000100:00080000:0.0:1787828486.530435:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1787828486.530438:0:37:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828486.530445:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 1438 00000100:00080000:0.0:1787828486.530448:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828486.530450:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:0.0:1787828486.530467:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787828486.530478:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c68f00 x1874672572538368/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1787828486.530495:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787828486.530538:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828486/real 1787828486] req@ffff8cb999c6b840 x1874672572538112/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828511 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828486.530558:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c6b840 x1874672572538112/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828511 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828486.530570:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828486.530576:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787828486.530580:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c6b840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572538112:192.168.203.129@tcp:250:kworker.0 00000100:00100000:1.0:1787828486.530605:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c69a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572538240:192.168.203.129@tcp:38:kworker.0 00000800:00000100:1.0:1787828486.530629:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787828486.530643:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828486/real 1787828486] req@ffff8cb999c69a40 x1874672572538240/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828511 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828486.530656:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c69a40 x1874672572538240/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828511 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828486.530666:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828486.530669:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787828486.530673:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c69a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572538240:192.168.203.129@tcp:38:kworker.0 00000100:00100000:1.0:1787828486.530679:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c68f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572538368:192.168.203.129@tcp:38:kworker.0 00000800:00000100:1.0:1787828486.530699:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787828486.530709:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828486/real 1787828486] req@ffff8cb999c68f00 x1874672572538368/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828511 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828486.530718:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c68f00 x1874672572538368/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828511 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828486.530733:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828486.530735:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787828486.530738:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c68f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572538368:192.168.203.129@tcp:38:kworker.0 00010000:00010000:1.0:1787828487.362424:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787828488.386481:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787828489.410390:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787828490.434493:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787828491.458525:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787828491.650389:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828491.650399:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787828491.650404:0:37:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828491.650414:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 1443 00000100:00080000:0.0:1787828491.650427:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828491.650435:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:0.0:1787828491.650471:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c6bc00 x1874672572538496/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828491.650508:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828491.650513:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787828491.650515:0:37:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828491.650519:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 1443 00000100:00080000:0.0:1787828491.650523:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828491.650527:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:0.0:1787828491.650537:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787828491.650560:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c69e00 x1874672572538624/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828491.650576:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828491.650583:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00100000:3.0:1787828491.650595:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c6bc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572538496:192.168.203.129@tcp:250:kworker.0 00000100:00080000:0.0:1787828491.650607:0:37:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828491.650622:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 1443 00000100:00080000:0.0:1787828491.650627:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828491.650641:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:0.0:1787828491.650649:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787828491.650684:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c6b480 x1874672572538752/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1787828491.650898:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:3.0:1787828491.651014:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828491/real 1787828491] req@ffff8cb999c6bc00 x1874672572538496/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828516 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828491.651034:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c6bc00 x1874672572538496/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828516 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828491.651045:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828491.651049:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:3.0:1787828491.651053:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c6bc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572538496:192.168.203.129@tcp:250:kworker.0 00000100:00100000:3.0:1787828491.651061:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c69e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572538624:192.168.203.129@tcp:38:kworker.0 00000800:00000100:3.0:1787828491.651084:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:3.0:1787828491.651096:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828491/real 1787828491] req@ffff8cb999c69e00 x1874672572538624/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828516 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828491.651107:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c69e00 x1874672572538624/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828516 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828491.651114:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828491.651117:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:3.0:1787828491.651120:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c69e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572538624:192.168.203.129@tcp:38:kworker.0 00000100:00100000:3.0:1787828491.651163:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c6b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572538752:192.168.203.129@tcp:38:kworker.0 00000800:00000100:3.0:1787828491.651209:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:3.0:1787828491.651221:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828491/real 1787828491] req@ffff8cb999c6b480 x1874672572538752/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828516 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828491.651306:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c6b480 x1874672572538752/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828516 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828491.651316:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828491.651321:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:3.0:1787828491.651352:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c6b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572538752:192.168.203.129@tcp:38:kworker.0 00010000:00010000:1.0:1787828492.482383:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787828493.506562:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787828494.530423:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787828495.554473:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787828496.578550:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787828496.770407:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828496.770420:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787828496.770426:0:37:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828496.770441:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 1448 00000100:00080000:0.0:1787828496.770457:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828496.770467:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:0.0:1787828496.770522:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c69680 x1874672572538880/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828496.770583:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828496.770588:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787828496.770591:0:37:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828496.770596:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 1448 00000100:00080000:0.0:1787828496.770601:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828496.770614:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:0.0:1787828496.770628:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787828496.770645:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c68000 x1874672572539008/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828496.770691:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828496.770696:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1787828496.770701:0:37:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1787828496.770707:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c69680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572538880:192.168.203.129@tcp:250:kworker.0 00000100:00080000:0.0:1787828496.770710:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 1448 00000100:00080000:0.0:1787828496.770721:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828496.770736:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:0.0:1787828496.770744:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787828496.770763:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c692c0 x1874672572539136/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1787828496.770843:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:3.0:1787828496.770942:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828496/real 1787828496] req@ffff8cb999c69680 x1874672572538880/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828521 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828496.770966:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c69680 x1874672572538880/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828521 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828496.770981:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828496.770988:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:3.0:1787828496.770994:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c69680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572538880:192.168.203.129@tcp:250:kworker.0 00000100:00100000:3.0:1787828496.771027:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c68000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572539008:192.168.203.129@tcp:38:kworker.0 00000800:00000100:3.0:1787828496.771055:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:3.0:1787828496.771069:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828496/real 1787828496] req@ffff8cb999c68000 x1874672572539008/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828521 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828496.771083:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c68000 x1874672572539008/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828521 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828496.771134:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828496.771139:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:3.0:1787828496.771143:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c68000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572539008:192.168.203.129@tcp:38:kworker.0 00000100:00100000:3.0:1787828496.771150:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c692c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572539136:192.168.203.129@tcp:38:kworker.0 00000800:00000100:3.0:1787828496.771179:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:3.0:1787828496.771193:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828496/real 1787828496] req@ffff8cb999c692c0 x1874672572539136/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828521 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828496.771206:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c692c0 x1874672572539136/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828521 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828496.771215:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828496.771218:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:3.0:1787828496.771221:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c692c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572539136:192.168.203.129@tcp:38:kworker.0 00010000:00010000:0.0:1787828497.602501:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787828498.626511:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787828499.650434:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787828500.674502:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787828501.698419:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787828501.890320:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828501.890331:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787828501.890335:0:37:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828501.890348:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 1454 00000100:00080000:0.0:1787828501.890361:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828501.890368:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:0.0:1787828501.890413:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c6a1c0 x1874672572539264/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828501.890440:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828501.890443:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787828501.890444:0:37:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828501.890447:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 1454 00000100:00080000:0.0:1787828501.890467:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828501.890473:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:3.0:1787828501.890475:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c6a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572539264:192.168.203.129@tcp:250:kworker.0 00000008:00100000:0.0:1787828501.890485:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787828501.890497:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c6a580 x1874672572539392/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828501.890505:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828501.890509:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1787828501.890510:0:37:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828501.890530:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 1454 00000100:00080000:0.0:1787828501.890533:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828501.890536:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:0.0:1787828501.890540:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787828501.890550:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c692c0 x1874672572539520/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1787828501.890570:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:3.0:1787828501.890629:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828501/real 1787828501] req@ffff8cb999c6a1c0 x1874672572539264/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828526 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828501.890648:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c6a1c0 x1874672572539264/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828526 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828501.890657:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828501.890660:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:3.0:1787828501.890669:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c6a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572539264:192.168.203.129@tcp:250:kworker.0 00000100:00100000:3.0:1787828501.890718:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c6a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572539392:192.168.203.129@tcp:38:kworker.0 00000800:00000100:3.0:1787828501.890748:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:3.0:1787828501.890763:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828501/real 1787828501] req@ffff8cb999c6a580 x1874672572539392/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828526 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828501.890775:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c6a580 x1874672572539392/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828526 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828501.890792:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828501.890803:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:3.0:1787828501.890806:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c6a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572539392:192.168.203.129@tcp:38:kworker.0 00000100:00100000:3.0:1787828501.890813:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c692c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572539520:192.168.203.129@tcp:38:kworker.0 00000800:00000100:3.0:1787828501.890830:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:3.0:1787828501.890840:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828501/real 1787828501] req@ffff8cb999c692c0 x1874672572539520/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828526 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828501.890852:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c692c0 x1874672572539520/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828526 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828501.890868:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828501.890872:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:3.0:1787828501.890879:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c692c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572539520:192.168.203.129@tcp:38:kworker.0 00010000:00010000:1.0:1787828502.722466:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787828503.746592:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787828504.770530:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787828505.794426:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787828506.818557:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787828507.010399:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828507.010414:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787828507.010445:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828507.010476:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 1459 00000100:00080000:3.0:1787828507.010493:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828507.010504:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:3.0:1787828507.010566:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb992317480 x1874672572539648/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828507.010602:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828507.010605:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787828507.010607:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828507.010610:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 1459 00000100:00080000:3.0:1787828507.010613:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828507.010616:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:3.0:1787828507.010626:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787828507.010635:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb992315e00 x1874672572539776/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828507.010644:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828507.010647:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1787828507.010649:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828507.010652:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 1459 00000100:00080000:3.0:1787828507.010655:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828507.010658:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:3.0:1787828507.010661:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787828507.010669:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb992316940 x1874672572539904/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787828507.010680:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb992317480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572539648:192.168.203.129@tcp:250:kworker.0 00000800:00000100:1.0:1787828507.010768:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787828507.010826:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828507/real 1787828507] req@ffff8cb992317480 x1874672572539648/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828532 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828507.010850:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb992317480 x1874672572539648/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828532 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828507.010865:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828507.010871:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787828507.010876:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb992317480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572539648:192.168.203.129@tcp:250:kworker.0 00000100:00100000:1.0:1787828507.010896:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb992315e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572539776:192.168.203.129@tcp:38:kworker.0 00000800:00000100:1.0:1787828507.010926:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787828507.010945:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828507/real 1787828507] req@ffff8cb992315e00 x1874672572539776/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828532 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828507.010962:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb992315e00 x1874672572539776/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828532 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828507.010973:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828507.010977:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787828507.010982:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb992315e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572539776:192.168.203.129@tcp:38:kworker.0 00000100:00100000:1.0:1787828507.011022:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb992316940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572539904:192.168.203.129@tcp:38:kworker.0 00000800:00000100:1.0:1787828507.011065:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787828507.011096:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828507/real 1787828507] req@ffff8cb992316940 x1874672572539904/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828532 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828507.011111:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb992316940 x1874672572539904/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828532 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828507.011122:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828507.011126:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787828507.011131:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb992316940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572539904:192.168.203.129@tcp:38:kworker.0 00010000:00010000:1.0:1787828507.842430:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787828508.866540:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787828509.890553:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787828510.914599:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787828511.938489:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787828512.130381:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828512.130394:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787828512.130400:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828512.130412:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 1464 00000100:00080000:3.0:1787828512.130425:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828512.130434:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:3.0:1787828512.130482:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb992314f00 x1874672572540032/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828512.130523:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828512.130531:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787828512.130535:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828512.130543:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 1464 00000100:00080000:3.0:1787828512.130549:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828512.130557:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:3.0:1787828512.130573:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787828512.130600:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb9923161c0 x1874672572540160/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787828512.130624:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb992314f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572540032:192.168.203.129@tcp:250:kworker.0 00000100:00080000:3.0:1787828512.130627:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828512.130641:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1787828512.130644:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828512.130651:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 1464 00000100:00080000:3.0:1787828512.130657:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828512.130662:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:3.0:1787828512.130692:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787828512.130713:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb992315a40 x1874672572540288/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787828512.130793:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb9923161c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572540160:192.168.203.129@tcp:38:kworker.0 00000100:00100000:1.0:1787828512.130832:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb992315a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572540288:192.168.203.129@tcp:38:kworker.0 00010000:00010000:0.0:1787828512.962507:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787828513.986472:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787828515.010826:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:0.0:1787828515.202668:0:2382:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.129:988 00000400:00000100:0.0:1787828515.202733:0:2382:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.129@tcp at host 192.168.203.129 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:0.0:1787828515.202753:0:2382:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.129@tcp 00000800:00000100:0.0:1787828515.202799:0:2382:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.129@tcp 00000800:00000100:0.0:1787828515.202811:0:2382:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.129@tcp 00000100:00100000:1.0:1787828515.202909:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828512/real 1787828515] req@ffff8cb9923161c0 x1874672572540160/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828537 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828515.202951:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb9923161c0 x1874672572540160/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828537 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828515.202966:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828515.202974:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787828515.203004:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb9923161c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572540160:192.168.203.129@tcp:38:kworker.0 00000100:00100000:1.0:1787828515.203017:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828512/real 1787828515] req@ffff8cb992315a40 x1874672572540288/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828537 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828515.203057:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb992315a40 x1874672572540288/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828537 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828515.203072:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828515.203078:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787828515.203084:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb992315a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572540288:192.168.203.129@tcp:38:kworker.0 00000100:00100000:1.0:1787828515.203096:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828512/real 1787828515] req@ffff8cb992314f00 x1874672572540032/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828537 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828515.203159:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb992314f00 x1874672572540032/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828537 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828515.203175:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828515.203183:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787828515.203189:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb992314f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572540032:192.168.203.129@tcp:250:kworker.0 00010000:00010000:0.0:1787828516.035117:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787828517.059005:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787828517.250388:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828517.250403:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787828517.250409:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828517.250424:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 1469 00000100:00080000:3.0:1787828517.250439:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828517.250451:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:3.0:1787828517.250495:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb9923170c0 x1874672572540416/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828517.250529:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828517.250533:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787828517.250535:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828517.250539:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 1469 00000100:00080000:3.0:1787828517.250543:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828517.250547:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:3.0:1787828517.250558:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787828517.250593:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb992314f00 x1874672572540544/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828517.250609:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828517.250613:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1787828517.250616:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1787828517.250625:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb9923170c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572540416:192.168.203.129@tcp:250:kworker.0 00000100:00080000:3.0:1787828517.250626:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 1469 00000100:00080000:3.0:1787828517.250652:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828517.250658:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:3.0:1787828517.250684:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787828517.250704:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb992315a40 x1874672572540672/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1787828517.250763:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787828517.250840:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828517/real 1787828517] req@ffff8cb9923170c0 x1874672572540416/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828542 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828517.250887:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb9923170c0 x1874672572540416/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828542 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828517.250903:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828517.250909:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787828517.250916:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb9923170c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572540416:192.168.203.129@tcp:250:kworker.0 00000100:00100000:1.0:1787828517.250953:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb992314f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572540544:192.168.203.129@tcp:38:kworker.0 00000800:00000100:1.0:1787828517.251015:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787828517.251037:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828517/real 1787828517] req@ffff8cb992314f00 x1874672572540544/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828542 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828517.251059:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb992314f00 x1874672572540544/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828542 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828517.251093:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828517.251099:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787828517.251105:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb992314f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572540544:192.168.203.129@tcp:38:kworker.0 00000100:00100000:1.0:1787828517.251115:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb992315a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572540672:192.168.203.129@tcp:38:kworker.0 00000800:00000100:1.0:1787828517.251159:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787828517.251179:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828517/real 1787828517] req@ffff8cb992315a40 x1874672572540672/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828542 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828517.251225:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb992315a40 x1874672572540672/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828542 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828517.251240:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828517.251246:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787828517.251252:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb992315a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572540672:192.168.203.129@tcp:38:kworker.0 00010000:00010000:1.0:1787828518.082873:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787828519.106758:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787828520.131038:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787828521.154658:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787828522.178541:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787828522.370415:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828522.370429:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787828522.370435:0:37:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828522.370448:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 1474 00000100:00080000:2.0:1787828522.370462:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828522.370473:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:2.0:1787828522.370521:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8ed00 x1874672572540800/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828522.370582:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828522.370589:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787828522.370592:0:37:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828522.370599:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 1474 00000100:00080000:2.0:1787828522.370606:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828522.370615:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:2.0:1787828522.370630:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787828522.370643:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8ed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572540800:192.168.203.129@tcp:250:kworker.0 00000100:00100000:2.0:1787828522.370658:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8f840 x1874672572540928/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828522.370680:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828522.370686:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1787828522.370690:0:37:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828522.370697:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 1474 00000100:00080000:2.0:1787828522.370703:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828522.370710:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:2.0:1787828522.370718:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000800:00000100:1.0:1787828522.370730:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787828522.370739:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8e940 x1874672572541056/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787828522.370796:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828522/real 1787828522] req@ffff8cb998a8ed00 x1874672572540800/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828547 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828522.370825:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8ed00 x1874672572540800/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828547 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828522.370863:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828522.370870:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787828522.370877:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8ed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572540800:192.168.203.129@tcp:250:kworker.0 00000100:00100000:1.0:1787828522.370912:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8f840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572540928:192.168.203.129@tcp:38:kworker.0 00000800:00000100:1.0:1787828522.370967:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787828522.370984:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828522/real 1787828522] req@ffff8cb998a8f840 x1874672572540928/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828547 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828522.370998:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8f840 x1874672572540928/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828547 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828522.371007:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828522.371010:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787828522.371014:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8f840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572540928:192.168.203.129@tcp:38:kworker.0 00000100:00100000:1.0:1787828522.371020:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8e940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572541056:192.168.203.129@tcp:38:kworker.0 00000800:00000100:1.0:1787828522.371039:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787828522.371049:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828522/real 1787828522] req@ffff8cb998a8e940 x1874672572541056/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828547 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828522.371060:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8e940 x1874672572541056/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828547 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828522.371068:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828522.371071:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787828522.371074:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8e940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572541056:192.168.203.129@tcp:38:kworker.0 00010000:00010000:1.0:1787828523.202493:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787828524.226817:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787828525.250469:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787828526.274548:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787828527.298490:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787828527.490477:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828527.490496:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787828527.490506:0:37:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828527.490526:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 1479 00000100:00080000:0.0:1787828527.490543:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828527.490555:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:0.0:1787828527.490625:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c68000 x1874672572541184/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828527.490745:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828527.490752:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787828527.490756:0:37:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828527.490768:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 1479 00000100:00080000:0.0:1787828527.490778:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828527.490786:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:0.0:1787828527.490804:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787828527.490827:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c69680 x1874672572541312/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828527.490845:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828527.490858:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00100000:1.0:1787828527.490859:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c68000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572541184:192.168.203.129@tcp:250:kworker.0 00000100:00080000:0.0:1787828527.490875:0:37:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828527.490918:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 1479 00000100:00080000:0.0:1787828527.490926:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828527.490933:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:0.0:1787828527.490975:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787828527.490997:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c6b480 x1874672572541440/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1787828527.491022:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787828527.491081:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828527/real 1787828527] req@ffff8cb999c68000 x1874672572541184/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828552 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828527.491102:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c68000 x1874672572541184/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828552 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828527.491116:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828527.491121:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787828527.491126:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c68000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572541184:192.168.203.129@tcp:250:kworker.0 00000100:00100000:1.0:1787828527.491155:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c69680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572541312:192.168.203.129@tcp:38:kworker.0 00000800:00000100:1.0:1787828527.491185:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787828527.491213:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828527/real 1787828527] req@ffff8cb999c69680 x1874672572541312/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828552 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828527.491223:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c69680 x1874672572541312/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828552 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828527.491229:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828527.491231:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787828527.491234:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c69680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572541312:192.168.203.129@tcp:38:kworker.0 00000100:00100000:1.0:1787828527.491251:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c6b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572541440:192.168.203.129@tcp:38:kworker.0 00000800:00000100:1.0:1787828527.491268:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787828527.491279:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828527/real 1787828527] req@ffff8cb999c6b480 x1874672572541440/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828552 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828527.491328:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c6b480 x1874672572541440/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828552 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828527.491340:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828527.491350:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787828527.491355:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c6b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572541440:192.168.203.129@tcp:38:kworker.0 00010000:00010000:0.0:1787828528.322486:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787828529.346460:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787828530.370419:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787828531.394422:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787828532.418414:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787828532.610455:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828532.610465:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787828532.610470:0:37:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828532.610479:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 1484 00000100:00080000:2.0:1787828532.610508:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828532.610516:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:2.0:1787828532.610556:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8cb40 x1874672572541568/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828532.610595:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828532.610600:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787828532.610603:0:37:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828532.610610:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 1484 00000100:00080000:2.0:1787828532.610616:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828532.610622:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:2.0:1787828532.610636:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787828532.610652:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8f480 x1874672572541696/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828532.610664:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828532.610669:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00100000:1.0:1787828532.610670:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8cb40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572541568:192.168.203.129@tcp:250:kworker.0 00000100:00080000:2.0:1787828532.610672:0:37:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828532.610683:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 1484 00000100:00080000:2.0:1787828532.610686:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828532.610689:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:2.0:1787828532.610694:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787828532.610716:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8c000 x1874672572541824/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1787828532.610758:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787828532.610822:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828532/real 1787828532] req@ffff8cb998a8cb40 x1874672572541568/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828557 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828532.610847:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8cb40 x1874672572541568/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828557 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828532.610914:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828532.610919:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787828532.610923:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8cb40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572541568:192.168.203.129@tcp:250:kworker.0 00000100:00100000:1.0:1787828532.610948:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572541696:192.168.203.129@tcp:38:kworker.0 00000800:00000100:1.0:1787828532.610984:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787828532.610998:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828532/real 1787828532] req@ffff8cb998a8f480 x1874672572541696/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828557 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828532.611010:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8f480 x1874672572541696/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828557 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828532.611018:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828532.611021:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787828532.611024:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572541696:192.168.203.129@tcp:38:kworker.0 00000100:00100000:1.0:1787828532.611030:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572541824:192.168.203.129@tcp:38:kworker.0 00000800:00000100:1.0:1787828532.611049:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787828532.611064:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828532/real 1787828532] req@ffff8cb998a8c000 x1874672572541824/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828557 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828532.611081:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8c000 x1874672572541824/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828557 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828532.611093:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828532.611098:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787828532.611104:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572541824:192.168.203.129@tcp:38:kworker.0 00010000:00010000:1.0:1787828533.442478:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787828534.466434:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787828535.490443:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787828536.514500:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787828537.538526:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787828537.730340:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828537.730350:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787828537.730354:0:37:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828537.730364:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 1489 00000100:00080000:0.0:1787828537.730375:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828537.730383:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:0.0:1787828537.730421:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c69e00 x1874672572541952/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828537.730443:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828537.730446:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787828537.730447:0:37:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828537.730450:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 1489 00000100:00080000:0.0:1787828537.730453:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828537.730455:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:0.0:1787828537.730463:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787828537.730470:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c6bc00 x1874672572542080/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828537.730477:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828537.730493:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1787828537.730495:0:37:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828537.730501:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 1489 00000100:00080000:0.0:1787828537.730503:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828537.730507:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:0.0:1787828537.730513:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787828537.730522:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c68f00 x1874672572542208/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787828537.730556:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c69e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572541952:192.168.203.129@tcp:250:kworker.0 00000800:00000100:1.0:1787828537.730687:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787828537.730775:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828537/real 1787828537] req@ffff8cb999c69e00 x1874672572541952/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828562 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828537.730801:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c69e00 x1874672572541952/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828562 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828537.730817:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828537.730849:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787828537.730855:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c69e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572541952:192.168.203.129@tcp:250:kworker.0 00000100:00100000:1.0:1787828537.730867:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c6bc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572542080:192.168.203.129@tcp:38:kworker.0 00000800:00000100:1.0:1787828537.730904:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787828537.730922:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828537/real 1787828537] req@ffff8cb999c6bc00 x1874672572542080/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828562 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828537.730938:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c6bc00 x1874672572542080/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828562 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828537.730949:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828537.730953:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787828537.730957:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c6bc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572542080:192.168.203.129@tcp:38:kworker.0 00000100:00100000:1.0:1787828537.730996:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c68f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572542208:192.168.203.129@tcp:38:kworker.0 00000800:00000100:1.0:1787828537.731018:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787828537.731030:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828537/real 1787828537] req@ffff8cb999c68f00 x1874672572542208/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828562 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828537.731045:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c68f00 x1874672572542208/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828562 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828537.731076:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828537.731081:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787828537.731085:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c68f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572542208:192.168.203.129@tcp:38:kworker.0 00010000:00010000:1.0:1787828538.562413:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787828539.586501:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787828540.610586:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787828541.634581:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787828542.658476:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787828542.850480:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828542.850495:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787828542.850502:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828542.850518:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 1495 00000100:00080000:3.0:1787828542.850546:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828542.850557:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:3.0:1787828542.850628:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb9923161c0 x1874672572542336/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828542.850662:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828542.850666:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787828542.850668:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828542.850682:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 1495 00000100:00080000:3.0:1787828542.850686:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828542.850689:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:3.0:1787828542.850718:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787828542.850733:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb992316940 x1874672572542464/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828542.850746:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1787828542.850754:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb9923161c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572542336:192.168.203.129@tcp:250:kworker.0 00000100:00080000:3.0:1787828542.850774:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1787828542.850777:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828542.850783:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 1495 00000100:00080000:3.0:1787828542.850826:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828542.850831:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:3.0:1787828542.850837:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787828542.850851:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb992315e00 x1874672572542592/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1787828542.850906:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787828542.850979:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828542/real 1787828542] req@ffff8cb9923161c0 x1874672572542336/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828567 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828542.851002:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb9923161c0 x1874672572542336/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828567 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828542.851020:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828542.851028:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787828542.851034:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb9923161c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572542336:192.168.203.129@tcp:250:kworker.0 00000100:00100000:1.0:1787828542.851089:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb992316940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572542464:192.168.203.129@tcp:38:kworker.0 00000800:00000100:1.0:1787828542.851134:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787828542.851147:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828542/real 1787828542] req@ffff8cb992316940 x1874672572542464/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828567 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828542.851158:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb992316940 x1874672572542464/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828567 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828542.851189:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828542.851192:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787828542.851195:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb992316940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572542464:192.168.203.129@tcp:38:kworker.0 00000100:00100000:1.0:1787828542.851201:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb992315e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572542592:192.168.203.129@tcp:38:kworker.0 00000800:00000100:1.0:1787828542.851220:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787828542.851229:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828542/real 1787828542] req@ffff8cb992315e00 x1874672572542592/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828567 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828542.851248:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb992315e00 x1874672572542592/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828567 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828542.851254:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828542.851256:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787828542.851259:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb992315e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572542592:192.168.203.129@tcp:38:kworker.0 00010000:00010000:0.0:1787828543.682458:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787828544.706471:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787828545.730527:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787828546.754469:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787828547.779468:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787828547.970415:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828547.970432:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787828547.970439:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828547.970456:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 1500 00000100:00080000:3.0:1787828547.970473:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828547.970485:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:3.0:1787828547.970542:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb992317480 x1874672572542720/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828547.970579:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828547.970584:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787828547.970586:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828547.970590:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 1500 00000100:00080000:3.0:1787828547.970594:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828547.970599:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:3.0:1787828547.970611:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787828547.970624:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb992314b40 x1874672572542848/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787828547.970643:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb992317480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572542720:192.168.203.129@tcp:250:kworker.0 00000100:00080000:3.0:1787828547.970671:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828547.970675:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1787828547.970677:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828547.970682:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 1500 00000100:00080000:3.0:1787828547.970686:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828547.970690:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:3.0:1787828547.970696:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787828547.970710:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb992314780 x1874672572542976/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1787828547.970771:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787828547.970867:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828547/real 1787828547] req@ffff8cb992317480 x1874672572542720/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828572 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828547.970892:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb992317480 x1874672572542720/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828572 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828547.970955:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828547.970960:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787828547.970964:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb992317480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572542720:192.168.203.129@tcp:250:kworker.0 00000100:00100000:1.0:1787828547.970998:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb992314b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572542848:192.168.203.129@tcp:38:kworker.0 00000800:00000100:1.0:1787828547.971024:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787828547.971038:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828547/real 1787828547] req@ffff8cb992314b40 x1874672572542848/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828572 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828547.971050:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb992314b40 x1874672572542848/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828572 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828547.971058:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828547.971063:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787828547.971069:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb992314b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572542848:192.168.203.129@tcp:38:kworker.0 00000100:00100000:1.0:1787828547.971078:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb992314780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572542976:192.168.203.129@tcp:38:kworker.0 00000800:00000100:1.0:1787828547.971108:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787828547.971126:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828547/real 1787828547] req@ffff8cb992314780 x1874672572542976/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828572 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828547.971146:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb992314780 x1874672572542976/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828572 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828547.971166:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828547.971170:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787828547.971173:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb992314780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572542976:192.168.203.129@tcp:38:kworker.0 00010000:00010000:1.0:1787828548.802532:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787828549.826427:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787828550.850416:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787828551.874514:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787828552.898394:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787828553.090437:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828553.090500:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787828553.090508:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828553.090528:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 1505 00000100:00080000:2.0:1787828553.090548:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828553.090559:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:2.0:1787828553.090627:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8d680 x1874672572543104/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828553.090669:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828553.090674:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787828553.090677:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828553.090682:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 1505 00000100:00080000:2.0:1787828553.090687:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828553.090691:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:2.0:1787828553.090706:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787828553.090761:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8e580 x1874672572543232/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828553.090779:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828553.090787:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00100000:1.0:1787828553.090788:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8d680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572543104:192.168.203.129@tcp:250:kworker.0 00000100:00080000:2.0:1787828553.090791:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828553.090807:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 1505 00000100:00080000:2.0:1787828553.090813:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828553.090819:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:2.0:1787828553.090826:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787828553.090843:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8c780 x1874672572543360/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1787828553.090901:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787828553.090964:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828553/real 1787828553] req@ffff8cb998a8d680 x1874672572543104/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828578 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828553.090989:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8d680 x1874672572543104/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828578 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828553.091004:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828553.091009:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787828553.091015:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8d680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572543104:192.168.203.129@tcp:250:kworker.0 00000100:00100000:1.0:1787828553.091047:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8e580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572543232:192.168.203.129@tcp:38:kworker.0 00000800:00000100:1.0:1787828553.091074:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787828553.091090:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828553/real 1787828553] req@ffff8cb998a8e580 x1874672572543232/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828578 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828553.091105:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8e580 x1874672572543232/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828578 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828553.091116:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828553.091121:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787828553.091125:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8e580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572543232:192.168.203.129@tcp:38:kworker.0 00000100:00100000:1.0:1787828553.091132:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8c780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572543360:192.168.203.129@tcp:38:kworker.0 00000800:00000100:1.0:1787828553.091171:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787828553.091185:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828553/real 1787828553] req@ffff8cb998a8c780 x1874672572543360/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828578 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828553.091200:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8c780 x1874672572543360/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828578 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828553.091210:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828553.091214:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787828553.091218:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8c780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572543360:192.168.203.129@tcp:38:kworker.0 00010000:00010000:1.0:1787828553.922505:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787828554.946470:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787828555.970465:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787828556.994524:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787828558.018470:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787828558.210379:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828558.210391:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787828558.210396:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828558.210407:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 1510 00000100:00080000:2.0:1787828558.210418:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828558.210427:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:2.0:1787828558.210479:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8cf00 x1874672572543488/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828558.210510:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828558.210515:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787828558.210518:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828558.210522:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 1510 00000100:00080000:2.0:1787828558.210537:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828558.210542:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:2.0:1787828558.210553:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787828558.210567:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8f0c0 x1874672572543616/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828558.210580:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828558.210584:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1787828558.210587:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828558.210591:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 1510 00000100:00080000:2.0:1787828558.210595:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1787828558.210610:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8cf00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572543488:192.168.203.129@tcp:250:kworker.0 00000100:00080000:2.0:1787828558.210614:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:2.0:1787828558.210639:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787828558.210657:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8fc00 x1874672572543744/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1787828558.210703:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787828558.210762:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828558/real 1787828558] req@ffff8cb998a8cf00 x1874672572543488/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828583 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828558.210783:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8cf00 x1874672572543488/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828583 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828558.210807:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828558.210812:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787828558.210815:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8cf00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572543488:192.168.203.129@tcp:250:kworker.0 00000100:00100000:1.0:1787828558.210831:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8f0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572543616:192.168.203.129@tcp:38:kworker.0 00000800:00000100:1.0:1787828558.210861:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787828558.210873:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828558/real 1787828558] req@ffff8cb998a8f0c0 x1874672572543616/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828583 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828558.210882:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8f0c0 x1874672572543616/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828583 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828558.210888:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828558.210890:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787828558.210893:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8f0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572543616:192.168.203.129@tcp:38:kworker.0 00000100:00100000:1.0:1787828558.210913:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8fc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572543744:192.168.203.129@tcp:38:kworker.0 00000800:00000100:1.0:1787828558.210925:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787828558.210942:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828558/real 1787828558] req@ffff8cb998a8fc00 x1874672572543744/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828583 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828558.210951:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8fc00 x1874672572543744/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828583 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828558.210957:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828558.210959:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787828558.210961:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8fc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572543744:192.168.203.129@tcp:38:kworker.0 00010000:00010000:0.0:1787828559.042454:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787828560.066442:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787828561.090710:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787828562.114474:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787828563.138471:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787828563.330444:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828563.330457:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787828563.330462:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828563.330476:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 1515 00000100:00080000:0.0:1787828563.330490:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828563.330510:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:0.0:1787828563.330551:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c69a40 x1874672572543872/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828563.330602:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828563.330607:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787828563.330630:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828563.330636:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 1515 00000100:00080000:0.0:1787828563.330668:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828563.330674:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:0.0:1787828563.330689:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787828563.330691:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c69a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572543872:192.168.203.129@tcp:250:kworker.0 00000100:00100000:0.0:1787828563.330717:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c6b840 x1874672572544000/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828563.330743:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828563.330747:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1787828563.330749:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828563.330756:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 1515 00000100:00080000:0.0:1787828563.330760:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828563.330765:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:0.0:1787828563.330771:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000800:00000100:2.0:1787828563.330777:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787828563.330797:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c68f00 x1874672572544128/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787828563.330865:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828563/real 1787828563] req@ffff8cb999c69a40 x1874672572543872/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828588 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828563.330891:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c69a40 x1874672572543872/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828588 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828563.330921:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828563.330927:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787828563.330932:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c69a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572543872:192.168.203.129@tcp:250:kworker.0 00000100:00100000:2.0:1787828563.330965:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c6b840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572544000:192.168.203.129@tcp:38:kworker.0 00000800:00000100:2.0:1787828563.331005:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787828563.331025:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828563/real 1787828563] req@ffff8cb999c6b840 x1874672572544000/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828588 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828563.331040:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c6b840 x1874672572544000/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828588 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828563.331049:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828563.331053:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787828563.331057:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c6b840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572544000:192.168.203.129@tcp:38:kworker.0 00000100:00100000:2.0:1787828563.331080:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c68f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572544128:192.168.203.129@tcp:38:kworker.0 00000800:00000100:2.0:1787828563.331150:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787828563.331172:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828563/real 1787828563] req@ffff8cb999c68f00 x1874672572544128/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828588 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828563.331183:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c68f00 x1874672572544128/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828588 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828563.331190:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828563.331193:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787828563.331196:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c68f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572544128:192.168.203.129@tcp:38:kworker.0 00010000:00010000:1.0:1787828564.162394:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787828565.186511:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787828566.210407:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787828567.234520:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787828568.258431:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787828568.450418:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828568.450431:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787828568.450436:0:37:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828568.450447:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 1520 00000100:00080000:0.0:1787828568.450463:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828568.450476:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:0.0:1787828568.450564:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c6bc00 x1874672572544256/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828568.450599:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828568.450604:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787828568.450606:0:37:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828568.450633:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 1520 00000100:00080000:0.0:1787828568.450640:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828568.450647:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:0.0:1787828568.450677:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787828568.450681:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c6bc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572544256:192.168.203.129@tcp:250:kworker.0 00000100:00100000:0.0:1787828568.450699:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c69e00 x1874672572544384/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828568.450715:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828568.450720:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1787828568.450723:0:37:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828568.450728:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 1520 00000100:00080000:0.0:1787828568.450733:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828568.450739:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:0.0:1787828568.450744:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787828568.450761:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c6b480 x1874672572544512/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1787828568.450789:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787828568.450862:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828568/real 1787828568] req@ffff8cb999c6bc00 x1874672572544256/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828593 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828568.450907:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c6bc00 x1874672572544256/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828593 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828568.450924:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828568.450930:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787828568.450936:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c6bc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572544256:192.168.203.129@tcp:250:kworker.0 00000100:00100000:2.0:1787828568.450987:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c69e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572544384:192.168.203.129@tcp:38:kworker.0 00000800:00000100:2.0:1787828568.451015:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787828568.451049:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828568/real 1787828568] req@ffff8cb999c69e00 x1874672572544384/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828593 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828568.451103:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c69e00 x1874672572544384/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828593 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828568.451112:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828568.451116:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787828568.451120:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c69e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572544384:192.168.203.129@tcp:38:kworker.0 00000100:00100000:2.0:1787828568.451127:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c6b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572544512:192.168.203.129@tcp:38:kworker.0 00000800:00000100:2.0:1787828568.451162:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787828568.451182:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828568/real 1787828568] req@ffff8cb999c6b480 x1874672572544512/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828593 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828568.451204:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c6b480 x1874672572544512/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828593 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828568.451219:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828568.451225:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787828568.451231:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c6b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572544512:192.168.203.129@tcp:38:kworker.0 00010000:00010000:1.0:1787828569.282427:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787828570.306405:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787828571.330483:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787828572.354413:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787828573.378433:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787828573.570389:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828573.570397:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787828573.570402:0:37:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828573.570410:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 1525 00000100:00080000:1.0:1787828573.570421:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828573.570428:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:1.0:1787828573.570484:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f38b40 x1874672572544640/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828573.570527:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828573.570532:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787828573.570536:0:37:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828573.570543:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 1525 00000100:00080000:1.0:1787828573.570550:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828573.570557:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:1.0:1787828573.570612:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787828573.570634:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f39e00 x1874672572544768/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828573.570653:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828573.570660:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1787828573.570664:0:37:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828573.570675:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 1525 00000100:00100000:2.0:1787828573.570676:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f38b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572544640:192.168.203.129@tcp:250:kworker.0 00000100:00080000:1.0:1787828573.570700:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828573.570707:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:1.0:1787828573.570715:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787828573.570733:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f3b840 x1874672572544896/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1787828573.570756:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787828573.570831:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828573/real 1787828573] req@ffff8cb999f38b40 x1874672572544640/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828598 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828573.570849:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f38b40 x1874672572544640/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828598 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828573.570874:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828573.570878:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787828573.570882:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f38b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572544640:192.168.203.129@tcp:250:kworker.0 00000100:00100000:2.0:1787828573.570889:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f39e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572544768:192.168.203.129@tcp:38:kworker.0 00000800:00000100:2.0:1787828573.570912:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787828573.570924:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828573/real 1787828573] req@ffff8cb999f39e00 x1874672572544768/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828598 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828573.570934:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f39e00 x1874672572544768/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828598 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828573.570942:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828573.570945:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787828573.570948:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f39e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572544768:192.168.203.129@tcp:38:kworker.0 00000100:00100000:2.0:1787828573.570975:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f3b840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572544896:192.168.203.129@tcp:38:kworker.0 00000800:00000100:2.0:1787828573.570999:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787828573.571015:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828573/real 1787828573] req@ffff8cb999f3b840 x1874672572544896/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828598 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828573.571032:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f3b840 x1874672572544896/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828598 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828573.571044:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828573.571049:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787828573.571054:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f3b840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572544896:192.168.203.129@tcp:38:kworker.0 00010000:00010000:0.0:1787828574.402425:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787828575.426520:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787828576.450429:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787828577.474535:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787828578.498511:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787828578.690454:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828578.690491:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787828578.690499:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828578.690513:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 1530 00000100:00080000:0.0:1787828578.690530:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828578.690568:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:0.0:1787828578.690629:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c69680 x1874672572545024/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828578.690673:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828578.690679:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787828578.690682:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828578.690687:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 1530 00000100:00080000:0.0:1787828578.690692:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828578.690698:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:0.0:1787828578.690713:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787828578.690738:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c68000 x1874672572545152/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828578.690752:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828578.690756:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1787828578.690760:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828578.690766:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 1530 00000100:00080000:0.0:1787828578.690769:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828578.690776:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:0.0:1787828578.690783:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787828578.690802:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c692c0 x1874672572545280/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787828578.690807:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c69680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572545024:192.168.203.129@tcp:250:kworker.0 00000100:00100000:2.0:1787828578.690980:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c68000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572545152:192.168.203.129@tcp:38:kworker.0 00000100:00100000:2.0:1787828578.691044:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c692c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572545280:192.168.203.129@tcp:38:kworker.0 00010000:00010000:1.0:1787828579.522422:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787828580.546518:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787828581.570739:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:3.0:1787828581.762643:0:2383:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.129:988 00000400:00000100:3.0:1787828581.762692:0:2383:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.129@tcp at host 192.168.203.129 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:3.0:1787828581.762711:0:2383:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.129@tcp 00000800:00000100:3.0:1787828581.762762:0:2383:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.129@tcp 00000800:00000100:3.0:1787828581.762776:0:2383:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.129@tcp 00000100:00100000:2.0:1787828581.762817:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828578/real 1787828581] req@ffff8cb999c692c0 x1874672572545280/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828603 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828581.762850:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c692c0 x1874672572545280/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828603 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828581.762863:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828581.762869:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787828581.762874:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c692c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572545280:192.168.203.129@tcp:38:kworker.0 00000100:00100000:2.0:1787828581.762883:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828578/real 1787828581] req@ffff8cb999c69680 x1874672572545024/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828603 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828581.762897:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c69680 x1874672572545024/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828603 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828581.762907:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828581.762911:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787828581.762916:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c69680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572545024:192.168.203.129@tcp:250:kworker.0 00000100:00100000:2.0:1787828581.762924:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828578/real 1787828581] req@ffff8cb999c68000 x1874672572545152/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828603 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828581.762969:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c68000 x1874672572545152/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828603 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828581.762979:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828581.762984:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787828581.762987:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c68000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572545152:192.168.203.129@tcp:38:kworker.0 00010000:00010000:0.0:1787828582.594982:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787828583.618930:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787828583.810424:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828583.810435:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787828583.810440:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828583.810450:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 1535 00000100:00080000:2.0:1787828583.810461:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828583.810468:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:2.0:1787828583.810529:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8e1c0 x1874672572545408/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828583.810583:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828583.810586:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787828583.810588:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828583.810591:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 1535 00000100:00080000:2.0:1787828583.810595:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828583.810599:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:2.0:1787828583.810610:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787828583.810628:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8da40 x1874672572545536/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787828583.810653:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8e1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572545408:192.168.203.129@tcp:250:kworker.0 00000100:00080000:2.0:1787828583.810679:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828583.810684:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1787828583.810687:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828583.810694:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 1535 00000100:00080000:2.0:1787828583.810701:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828583.810706:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:2.0:1787828583.810714:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787828583.810732:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8de00 x1874672572545664/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1787828583.810739:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787828583.810795:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828583/real 1787828583] req@ffff8cb998a8e1c0 x1874672572545408/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828608 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828583.810821:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8e1c0 x1874672572545408/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828608 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828583.810836:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828583.810842:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787828583.810847:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8e1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572545408:192.168.203.129@tcp:250:kworker.0 00000100:00100000:1.0:1787828583.810879:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8da40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572545536:192.168.203.129@tcp:38:kworker.0 00000800:00000100:1.0:1787828583.810909:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787828583.810925:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828583/real 1787828583] req@ffff8cb998a8da40 x1874672572545536/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828608 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828583.810940:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8da40 x1874672572545536/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828608 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828583.810969:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828583.810974:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787828583.810978:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8da40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572545536:192.168.203.129@tcp:38:kworker.0 00000100:00100000:1.0:1787828583.810987:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8de00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572545664:192.168.203.129@tcp:38:kworker.0 00000800:00000100:1.0:1787828583.811015:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787828583.811031:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828583/real 1787828583] req@ffff8cb998a8de00 x1874672572545664/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828608 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828583.811047:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8de00 x1874672572545664/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828608 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828583.811057:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828583.811061:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787828583.811065:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8de00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572545664:192.168.203.129@tcp:38:kworker.0 00010000:00010000:2.0:1787828584.642616:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787828585.666472:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787828586.690654:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787828587.714574:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787828588.738781:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787828588.930377:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828588.930385:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787828588.930389:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828588.930398:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 1541 00000100:00080000:2.0:1787828588.930407:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828588.930413:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:2.0:1787828588.930445:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8d2c0 x1874672572545792/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828588.930481:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828588.930485:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787828588.930486:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828588.930505:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 1541 00000100:00080000:2.0:1787828588.930532:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828588.930537:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:2.0:1787828588.930546:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787828588.930556:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8c3c0 x1874672572545920/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787828588.930570:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8d2c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572545792:192.168.203.129@tcp:250:kworker.0 00000100:00080000:2.0:1787828588.930570:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828588.930591:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1787828588.930594:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828588.930598:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 1541 00000100:00080000:2.0:1787828588.930601:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828588.930604:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:2.0:1787828588.930610:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787828588.930625:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8de00 x1874672572546048/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1787828588.930665:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787828588.930742:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828588/real 1787828588] req@ffff8cb998a8d2c0 x1874672572545792/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828613 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828588.930764:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8d2c0 x1874672572545792/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828613 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828588.930777:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828588.930780:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787828588.930798:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8d2c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572545792:192.168.203.129@tcp:250:kworker.0 00000100:00100000:1.0:1787828588.930826:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8c3c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572545920:192.168.203.129@tcp:38:kworker.0 00000800:00000100:1.0:1787828588.930847:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787828588.930856:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828588/real 1787828588] req@ffff8cb998a8c3c0 x1874672572545920/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828613 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828588.930869:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8c3c0 x1874672572545920/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828613 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828588.930878:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828588.930881:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787828588.930884:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8c3c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572545920:192.168.203.129@tcp:38:kworker.0 00000100:00100000:1.0:1787828588.930900:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8de00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572546048:192.168.203.129@tcp:38:kworker.0 00000800:00000100:1.0:1787828588.930922:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787828588.930932:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828588/real 1787828588] req@ffff8cb998a8de00 x1874672572546048/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828613 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828588.930940:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8de00 x1874672572546048/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828613 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828588.930946:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828588.930948:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787828588.930950:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8de00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572546048:192.168.203.129@tcp:38:kworker.0 00010000:00010000:1.0:1787828589.762757:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787828590.786539:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787828591.810435:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787828592.834425:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787828593.858461:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787828594.050386:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828594.050394:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787828594.050398:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828594.050406:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 1546 00000100:00080000:0.0:1787828594.050415:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828594.050421:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:0.0:1787828594.050469:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c6a580 x1874672572546176/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828594.050491:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828594.050494:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787828594.050496:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828594.050499:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 1546 00000100:00080000:0.0:1787828594.050502:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828594.050504:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:0.0:1787828594.050513:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787828594.050521:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c6a1c0 x1874672572546304/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828594.050529:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828594.050534:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00100000:1.0:1787828594.050535:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c6a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572546176:192.168.203.129@tcp:250:kworker.0 00000100:00080000:0.0:1787828594.050549:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828594.050555:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 1546 00000100:00080000:0.0:1787828594.050558:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828594.050561:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:0.0:1787828594.050566:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787828594.050577:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c68000 x1874672572546432/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1787828594.050593:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787828594.050627:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828594/real 1787828594] req@ffff8cb999c6a580 x1874672572546176/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828619 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828594.050640:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c6a580 x1874672572546176/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828619 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828594.050649:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828594.050652:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787828594.050654:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c6a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572546176:192.168.203.129@tcp:250:kworker.0 00000100:00100000:1.0:1787828594.050672:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c6a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572546304:192.168.203.129@tcp:38:kworker.0 00000800:00000100:1.0:1787828594.050686:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787828594.050694:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828594/real 1787828594] req@ffff8cb999c6a1c0 x1874672572546304/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828619 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828594.050702:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c6a1c0 x1874672572546304/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828619 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828594.050708:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828594.050710:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787828594.050713:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c6a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572546304:192.168.203.129@tcp:38:kworker.0 00000100:00100000:1.0:1787828594.050716:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c68000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572546432:192.168.203.129@tcp:38:kworker.0 00000800:00000100:1.0:1787828594.050725:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787828594.050731:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828594/real 1787828594] req@ffff8cb999c68000 x1874672572546432/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828619 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828594.050746:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c68000 x1874672572546432/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828619 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828594.050752:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828594.050754:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787828594.050756:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c68000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572546432:192.168.203.129@tcp:38:kworker.0 00010000:00010000:1.0:1787828594.882390:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787828595.906492:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787828596.930422:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787828597.954487:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787828598.978417:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787828599.170426:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828599.170433:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787828599.170438:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828599.170449:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 1551 00000100:00080000:2.0:1787828599.170460:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828599.170468:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:2.0:1787828599.170506:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8da40 x1874672572546560/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828599.170547:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828599.170550:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787828599.170552:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828599.170555:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 1551 00000100:00080000:2.0:1787828599.170558:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828599.170562:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:2.0:1787828599.170573:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787828599.170584:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8da40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572546560:192.168.203.129@tcp:250:kworker.0 00000100:00100000:2.0:1787828599.170586:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8e1c0 x1874672572546688/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828599.170598:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828599.170601:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1787828599.170603:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828599.170606:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 1551 00000100:00080000:2.0:1787828599.170609:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828599.170613:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:2.0:1787828599.170618:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787828599.170633:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8fc00 x1874672572546816/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1787828599.170660:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787828599.170701:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828599/real 1787828599] req@ffff8cb998a8da40 x1874672572546560/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828624 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828599.170716:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8da40 x1874672572546560/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828624 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828599.170726:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828599.170729:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787828599.170740:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8da40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572546560:192.168.203.129@tcp:250:kworker.0 00000100:00100000:0.0:1787828599.170765:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8e1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572546688:192.168.203.129@tcp:38:kworker.0 00000800:00000100:0.0:1787828599.170782:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787828599.170792:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828599/real 1787828599] req@ffff8cb998a8e1c0 x1874672572546688/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828624 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828599.170801:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8e1c0 x1874672572546688/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828624 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828599.170817:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828599.170820:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787828599.170823:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8e1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572546688:192.168.203.129@tcp:38:kworker.0 00000100:00100000:0.0:1787828599.170827:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8fc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572546816:192.168.203.129@tcp:38:kworker.0 00000800:00000100:0.0:1787828599.170846:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787828599.170858:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828599/real 1787828599] req@ffff8cb998a8fc00 x1874672572546816/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828624 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828599.170871:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8fc00 x1874672572546816/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828624 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828599.170880:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828599.170883:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787828599.170887:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8fc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572546816:192.168.203.129@tcp:38:kworker.0 00010000:00010000:1.0:1787828600.002461:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787828601.026413:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787828602.050463:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787828603.074471:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787828604.098393:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787828604.290335:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828604.290343:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787828604.290347:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828604.290355:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 1556 00000100:00080000:2.0:1787828604.290396:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828604.290402:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:2.0:1787828604.290431:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8f0c0 x1874672572546944/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828604.290456:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828604.290460:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787828604.290462:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828604.290467:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 1556 00000100:00080000:2.0:1787828604.290472:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828604.290476:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:2.0:1787828604.290486:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787828604.290489:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8f0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572546944:192.168.203.129@tcp:250:kworker.0 00000100:00100000:2.0:1787828604.290502:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8cf00 x1874672572547072/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828604.290520:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828604.290523:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1787828604.290525:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828604.290528:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 1556 00000100:00080000:2.0:1787828604.290531:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828604.290534:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:2.0:1787828604.290539:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000800:00000100:0.0:1787828604.290547:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787828604.290551:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8c780 x1874672572547200/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787828604.290591:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828604/real 1787828604] req@ffff8cb998a8f0c0 x1874672572546944/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828629 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828604.290606:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8f0c0 x1874672572546944/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828629 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828604.290614:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828604.290617:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787828604.290628:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8f0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572546944:192.168.203.129@tcp:250:kworker.0 00000100:00100000:0.0:1787828604.290648:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8cf00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572547072:192.168.203.129@tcp:38:kworker.0 00000800:00000100:0.0:1787828604.290665:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787828604.290674:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828604/real 1787828604] req@ffff8cb998a8cf00 x1874672572547072/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828629 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828604.290682:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8cf00 x1874672572547072/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828629 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828604.290688:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828604.290690:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787828604.290692:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8cf00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572547072:192.168.203.129@tcp:38:kworker.0 00000100:00100000:0.0:1787828604.290696:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8c780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572547200:192.168.203.129@tcp:38:kworker.0 00000800:00000100:0.0:1787828604.290706:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787828604.290712:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828604/real 1787828604] req@ffff8cb998a8c780 x1874672572547200/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828629 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828604.290719:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8c780 x1874672572547200/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828629 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828604.290724:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828604.290726:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787828604.290728:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8c780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572547200:192.168.203.129@tcp:38:kworker.0 00010000:00010000:1.0:1787828605.122468:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787828606.146485:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787828607.170460:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787828608.194499:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787828609.218396:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787828609.410345:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828609.410354:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787828609.410368:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828609.410376:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 1561 00000100:00080000:0.0:1787828609.410384:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828609.410390:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:0.0:1787828609.410417:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c69680 x1874672572547328/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828609.410441:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828609.410444:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787828609.410446:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828609.410448:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 1561 00000100:00080000:0.0:1787828609.410451:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828609.410453:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:0.0:1787828609.410461:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787828609.410469:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c692c0 x1874672572547456/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828609.410477:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828609.410480:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1787828609.410482:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828609.410485:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 1561 00000100:00080000:0.0:1787828609.410487:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828609.410490:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:0.0:1787828609.410495:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787828609.410497:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c69680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572547328:192.168.203.129@tcp:250:kworker.0 00000100:00100000:0.0:1787828609.410504:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c6b480 x1874672572547584/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1787828609.410567:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787828609.410608:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828609/real 1787828609] req@ffff8cb999c69680 x1874672572547328/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828634 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828609.410622:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c69680 x1874672572547328/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828634 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828609.410637:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828609.410640:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787828609.410643:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c69680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572547328:192.168.203.129@tcp:250:kworker.0 00000100:00100000:1.0:1787828609.410648:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c692c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572547456:192.168.203.129@tcp:38:kworker.0 00000800:00000100:1.0:1787828609.410665:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787828609.410674:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828609/real 1787828609] req@ffff8cb999c692c0 x1874672572547456/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828634 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828609.410682:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c692c0 x1874672572547456/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828634 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828609.410687:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828609.410690:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787828609.410692:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c692c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572547456:192.168.203.129@tcp:38:kworker.0 00000100:00100000:1.0:1787828609.410708:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c6b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572547584:192.168.203.129@tcp:38:kworker.0 00000800:00000100:1.0:1787828609.410719:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787828609.410726:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828609/real 1787828609] req@ffff8cb999c6b480 x1874672572547584/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828634 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828609.410733:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c6b480 x1874672572547584/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828634 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828609.410738:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828609.410740:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787828609.410742:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c6b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572547584:192.168.203.129@tcp:38:kworker.0 00010000:00010000:1.0:1787828610.242377:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787828611.266380:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787828612.290427:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787828613.314436:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787828614.338407:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787828614.530390:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828614.530400:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787828614.530403:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828614.530411:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 1566 00000100:00080000:1.0:1787828614.530422:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828614.530428:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:1.0:1787828614.530459:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f39a40 x1874672572547712/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828614.530485:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828614.530488:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787828614.530489:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828614.530492:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 1566 00000100:00080000:1.0:1787828614.530496:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828614.530499:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:1.0:1787828614.530506:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787828614.530516:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f3b840 x1874672572547840/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828614.530527:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828614.530530:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00100000:0.0:1787828614.530538:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f39a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572547712:192.168.203.129@tcp:250:kworker.0 00000100:00080000:1.0:1787828614.530549:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828614.530558:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 1566 00000100:00080000:1.0:1787828614.530564:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828614.530569:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:1.0:1787828614.530577:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787828614.530597:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f39e00 x1874672572547968/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1787828614.530629:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787828614.530689:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828614/real 1787828614] req@ffff8cb999f39a40 x1874672572547712/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828639 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828614.530709:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f39a40 x1874672572547712/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828639 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828614.530719:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828614.530722:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787828614.530726:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f39a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572547712:192.168.203.129@tcp:250:kworker.0 00000100:00100000:0.0:1787828614.530751:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f3b840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572547840:192.168.203.129@tcp:38:kworker.0 00000800:00000100:0.0:1787828614.530769:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787828614.530779:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828614/real 1787828614] req@ffff8cb999f3b840 x1874672572547840/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828639 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828614.530787:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f3b840 x1874672572547840/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828639 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828614.530792:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828614.530795:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787828614.530797:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f3b840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572547840:192.168.203.129@tcp:38:kworker.0 00000100:00100000:0.0:1787828614.530800:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f39e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572547968:192.168.203.129@tcp:38:kworker.0 00000800:00000100:0.0:1787828614.530811:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787828614.530830:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828614/real 1787828614] req@ffff8cb999f39e00 x1874672572547968/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828639 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828614.530838:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f39e00 x1874672572547968/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828639 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828614.530843:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828614.530845:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787828614.530848:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f39e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572547968:192.168.203.129@tcp:38:kworker.0 00010000:00010000:1.0:1787828615.362409:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787828616.386412:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787828617.410377:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787828618.434399:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787828619.458417:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787828619.650327:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828619.650341:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787828619.650344:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828619.650352:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 1571 00000100:00080000:1.0:1787828619.650361:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828619.650366:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:1.0:1787828619.650397:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f38b40 x1874672572548096/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828619.650418:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828619.650421:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787828619.650422:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828619.650425:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 1571 00000100:00080000:1.0:1787828619.650427:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828619.650441:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:1.0:1787828619.650450:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787828619.650463:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f38f00 x1874672572548224/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828619.650471:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828619.650474:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1787828619.650475:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828619.650478:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 1571 00000100:00080000:1.0:1787828619.650480:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828619.650483:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:1.0:1787828619.650487:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787828619.650495:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f3b480 x1874672572548352/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787828619.650502:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f38b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572548096:192.168.203.129@tcp:250:kworker.0 00000800:00000100:0.0:1787828619.650596:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787828619.650648:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828619/real 1787828619] req@ffff8cb999f38b40 x1874672572548096/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828644 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828619.650670:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f38b40 x1874672572548096/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828644 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828619.650684:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828619.650689:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787828619.650694:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f38b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572548096:192.168.203.129@tcp:250:kworker.0 00000100:00100000:0.0:1787828619.650703:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f38f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572548224:192.168.203.129@tcp:38:kworker.0 00000800:00000100:0.0:1787828619.650727:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787828619.650742:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828619/real 1787828619] req@ffff8cb999f38f00 x1874672572548224/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828644 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828619.650756:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f38f00 x1874672572548224/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828644 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828619.650784:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828619.650788:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787828619.650793:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f38f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572548224:192.168.203.129@tcp:38:kworker.0 00000100:00100000:0.0:1787828619.650828:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f3b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572548352:192.168.203.129@tcp:38:kworker.0 00000800:00000100:0.0:1787828619.650852:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787828619.650865:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828619/real 1787828619] req@ffff8cb999f3b480 x1874672572548352/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828644 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828619.650878:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f3b480 x1874672572548352/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828644 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828619.650887:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828619.650891:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787828619.650895:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f3b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572548352:192.168.203.129@tcp:38:kworker.0 00010000:00010000:1.0:1787828620.482368:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787828621.506376:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787828622.530381:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787828623.554369:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787828624.578409:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787828624.770345:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828624.770353:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787828624.770356:0:37:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828624.770364:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 1576 00000100:00080000:1.0:1787828624.770372:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828624.770378:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:1.0:1787828624.770409:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f3b0c0 x1874672572548480/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828624.770448:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828624.770451:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787828624.770452:0:37:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828624.770455:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 1576 00000100:00080000:1.0:1787828624.770458:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828624.770461:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:1.0:1787828624.770469:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787828624.770480:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f392c0 x1874672572548608/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828624.770492:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828624.770496:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1787828624.770498:0:37:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828624.770503:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 1576 00000100:00100000:0.0:1787828624.770510:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f3b0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572548480:192.168.203.129@tcp:250:kworker.0 00000100:00080000:1.0:1787828624.770511:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828624.770528:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:1.0:1787828624.770535:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787828624.770550:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f3a580 x1874672572548736/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1787828624.770597:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787828624.770653:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828624/real 1787828624] req@ffff8cb999f3b0c0 x1874672572548480/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828649 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828624.770687:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f3b0c0 x1874672572548480/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828649 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828624.770700:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828624.770706:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787828624.770723:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f3b0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572548480:192.168.203.129@tcp:250:kworker.0 00000100:00100000:0.0:1787828624.770756:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f392c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572548608:192.168.203.129@tcp:38:kworker.0 00000800:00000100:0.0:1787828624.770783:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787828624.770798:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828624/real 1787828624] req@ffff8cb999f392c0 x1874672572548608/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828649 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828624.770806:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f392c0 x1874672572548608/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828649 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828624.770812:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828624.770814:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787828624.770824:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f392c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572548608:192.168.203.129@tcp:38:kworker.0 00000100:00100000:0.0:1787828624.770828:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f3a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572548736:192.168.203.129@tcp:38:kworker.0 00000800:00000100:0.0:1787828624.770843:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787828624.770850:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828624/real 1787828624] req@ffff8cb999f3a580 x1874672572548736/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828649 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828624.770858:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f3a580 x1874672572548736/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828649 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828624.770863:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828624.770865:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787828624.770867:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f3a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572548736:192.168.203.129@tcp:38:kworker.0 00010000:00010000:1.0:1787828625.602426:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787828626.626394:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787828627.650490:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787828628.674393:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787828629.698377:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787828629.890352:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828629.890390:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787828629.890394:0:37:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828629.890404:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 1582 00000100:00080000:2.0:1787828629.890415:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828629.890423:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:2.0:1787828629.890467:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8e580 x1874672572548864/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828629.890496:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828629.890500:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787828629.890502:0:37:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828629.890505:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 1582 00000100:00080000:2.0:1787828629.890509:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828629.890513:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:2.0:1787828629.890523:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787828629.890537:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8e580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572548864:192.168.203.129@tcp:250:kworker.0 00000100:00100000:2.0:1787828629.890548:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8d680 x1874672572548992/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828629.890562:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828629.890567:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1787828629.890569:0:37:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828629.890574:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 1582 00000100:00080000:2.0:1787828629.890578:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828629.890584:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:2.0:1787828629.890590:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000800:00000100:0.0:1787828629.890596:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787828629.890606:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8c000 x1874672572549120/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787828629.890653:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828629/real 1787828629] req@ffff8cb998a8e580 x1874672572548864/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828654 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828629.890670:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8e580 x1874672572548864/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828654 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828629.890683:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828629.890688:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787828629.890693:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8e580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572548864:192.168.203.129@tcp:250:kworker.0 00000100:00100000:0.0:1787828629.890719:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8d680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572548992:192.168.203.129@tcp:38:kworker.0 00000800:00000100:0.0:1787828629.890744:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787828629.890758:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828629/real 1787828629] req@ffff8cb998a8d680 x1874672572548992/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828654 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828629.890767:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8d680 x1874672572548992/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828654 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828629.890774:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828629.890776:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787828629.890778:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8d680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572548992:192.168.203.129@tcp:38:kworker.0 00000100:00100000:0.0:1787828629.890782:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572549120:192.168.203.129@tcp:38:kworker.0 00000800:00000100:0.0:1787828629.890796:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787828629.890803:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828629/real 1787828629] req@ffff8cb998a8c000 x1874672572549120/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828654 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828629.890811:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8c000 x1874672572549120/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828654 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828629.890826:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828629.890828:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787828629.890831:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572549120:192.168.203.129@tcp:38:kworker.0 00010000:00010000:1.0:1787828630.722389:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787828631.746463:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787828632.770427:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787828633.794404:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787828634.818461:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787828635.010340:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828635.010349:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787828635.010352:0:37:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828635.010360:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 1587 00000100:00080000:2.0:1787828635.010369:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828635.010375:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:2.0:1787828635.010406:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8f480 x1874672572549248/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828635.010429:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828635.010432:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787828635.010434:0:37:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828635.010436:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 1587 00000100:00080000:2.0:1787828635.010439:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828635.010441:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:2.0:1787828635.010449:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787828635.010458:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8cb40 x1874672572549376/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828635.010466:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828635.010468:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1787828635.010470:0:37:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828635.010472:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 1587 00000100:00080000:2.0:1787828635.010473:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828635.010475:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:2.0:1787828635.010491:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787828635.010499:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8e940 x1874672572549504/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787828635.010510:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572549248:192.168.203.129@tcp:250:kworker.0 00000800:00000100:0.0:1787828635.010566:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787828635.010609:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828635/real 1787828635] req@ffff8cb998a8f480 x1874672572549248/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828660 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828635.010624:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8f480 x1874672572549248/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828660 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828635.010633:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828635.010637:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787828635.010641:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572549248:192.168.203.129@tcp:250:kworker.0 00000100:00100000:0.0:1787828635.010647:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8cb40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572549376:192.168.203.129@tcp:38:kworker.0 00000800:00000100:0.0:1787828635.010670:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787828635.010681:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828635/real 1787828635] req@ffff8cb998a8cb40 x1874672572549376/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828660 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828635.010690:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8cb40 x1874672572549376/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828660 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828635.010696:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828635.010698:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787828635.010701:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8cb40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572549376:192.168.203.129@tcp:38:kworker.0 00000100:00100000:0.0:1787828635.010734:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8e940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572549504:192.168.203.129@tcp:38:kworker.0 00000800:00000100:0.0:1787828635.010752:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787828635.010762:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828635/real 1787828635] req@ffff8cb998a8e940 x1874672572549504/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828660 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828635.010771:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8e940 x1874672572549504/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828660 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828635.010786:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828635.010790:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787828635.010793:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8e940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572549504:192.168.203.129@tcp:38:kworker.0 00010000:00010000:1.0:1787828635.842582:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787828636.866439:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787828637.890493:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787828638.914429:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787828639.938412:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787828640.130369:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828640.130379:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787828640.130382:0:37:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828640.130390:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 1592 00000100:00080000:1.0:1787828640.130399:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828640.130405:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:1.0:1787828640.130434:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f383c0 x1874672572549632/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828640.130459:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828640.130462:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787828640.130463:0:37:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828640.130466:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 1592 00000100:00080000:1.0:1787828640.130468:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828640.130471:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:1.0:1787828640.130479:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787828640.130492:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f3a580 x1874672572549760/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787828640.130493:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f383c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572549632:192.168.203.129@tcp:250:kworker.0 00000100:00080000:1.0:1787828640.130525:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828640.130527:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1787828640.130529:0:37:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828640.130533:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 1592 00000100:00080000:1.0:1787828640.130545:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828640.130548:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000800:00000100:0.0:1787828640.130553:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000008:00100000:1.0:1787828640.130553:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787828640.130566:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f392c0 x1874672572549888/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787828640.130589:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828640/real 1787828640] req@ffff8cb999f383c0 x1874672572549632/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828665 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828640.130603:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f383c0 x1874672572549632/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828665 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828640.130611:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828640.130614:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787828640.130617:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f383c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572549632:192.168.203.129@tcp:250:kworker.0 00000100:00100000:0.0:1787828640.130643:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f3a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572549760:192.168.203.129@tcp:38:kworker.0 00000800:00000100:0.0:1787828640.130659:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787828640.130668:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828640/real 1787828640] req@ffff8cb999f3a580 x1874672572549760/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828665 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828640.130684:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f3a580 x1874672572549760/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828665 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828640.130690:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828640.130692:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787828640.130694:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f3a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572549760:192.168.203.129@tcp:38:kworker.0 00000100:00100000:0.0:1787828640.130698:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f392c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572549888:192.168.203.129@tcp:38:kworker.0 00000800:00000100:0.0:1787828640.130711:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787828640.130718:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828640/real 1787828640] req@ffff8cb999f392c0 x1874672572549888/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828665 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828640.130725:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f392c0 x1874672572549888/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828665 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828640.130730:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828640.130732:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787828640.130733:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f392c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572549888:192.168.203.129@tcp:38:kworker.0 00010000:00010000:1.0:1787828640.962410:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787828641.986400:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787828643.010477:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787828644.034414:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787828645.058459:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787828645.250313:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828645.250321:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787828645.250324:0:37:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828645.250332:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 1597 00000100:00080000:1.0:1787828645.250340:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828645.250352:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:1.0:1787828645.250384:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f3b0c0 x1874672572550016/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828645.250406:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828645.250409:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787828645.250422:0:37:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828645.250425:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 1597 00000100:00080000:1.0:1787828645.250429:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828645.250432:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:1.0:1787828645.250440:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787828645.250451:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f3b480 x1874672572550144/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828645.250459:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828645.250462:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1787828645.250463:0:37:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828645.250466:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 1597 00000100:00080000:1.0:1787828645.250468:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828645.250470:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:1.0:1787828645.250473:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787828645.250481:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f3b0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572550016:192.168.203.129@tcp:250:kworker.0 00000100:00100000:1.0:1787828645.250490:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f38f00 x1874672572550272/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787828645.250604:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f3b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572550144:192.168.203.129@tcp:38:kworker.0 00000100:00100000:0.0:1787828645.250634:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f38f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572550272:192.168.203.129@tcp:38:kworker.0 00010000:00010000:1.0:1787828646.082444:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787828647.106426:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787828648.130848:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:2.0:1787828648.322604:0:2384:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.129:988 00000400:00000100:2.0:1787828648.322653:0:2384:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.129@tcp at host 192.168.203.129 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:2.0:1787828648.322671:0:2384:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.129@tcp 00000800:00000100:2.0:1787828648.322712:0:2384:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.129@tcp 00000800:00000100:2.0:1787828648.322722:0:2384:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.129@tcp 00000100:00100000:0.0:1787828648.322774:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828645/real 1787828648] req@ffff8cb999f38f00 x1874672572550272/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828670 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828648.322825:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f38f00 x1874672572550272/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828670 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828648.322840:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828648.322847:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787828648.322852:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f38f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572550272:192.168.203.129@tcp:38:kworker.0 00000100:00100000:0.0:1787828648.322861:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828645/real 1787828648] req@ffff8cb999f3b0c0 x1874672572550016/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828670 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828648.322877:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f3b0c0 x1874672572550016/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828670 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828648.322887:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828648.322891:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787828648.322895:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f3b0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572550016:192.168.203.129@tcp:250:kworker.0 00000100:00100000:0.0:1787828648.322903:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828645/real 1787828648] req@ffff8cb999f3b480 x1874672572550144/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828670 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828648.322916:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f3b480 x1874672572550144/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828670 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828648.322924:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828648.322927:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787828648.322930:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f3b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572550144:192.168.203.129@tcp:38:kworker.0 00010000:00010000:1.0:1787828649.154730:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787828650.178455:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787828650.370407:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828650.370416:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787828650.370420:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828650.370429:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 1602 00000100:00080000:3.0:1787828650.370438:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828650.370445:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:3.0:1787828650.370475:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb992316580 x1874672572550400/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828650.370511:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828650.370530:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787828650.370533:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828650.370538:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 1602 00000100:00080000:3.0:1787828650.370544:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828650.370577:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:3.0:1787828650.370587:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787828650.370597:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb992316580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572550400:192.168.203.129@tcp:250:kworker.0 00000100:00100000:3.0:1787828650.370605:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb992314780 x1874672572550528/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828650.370625:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828650.370629:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1787828650.370631:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828650.370636:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 1602 00000100:00080000:3.0:1787828650.370640:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828650.370644:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:3.0:1787828650.370650:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787828650.370664:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb992314b40 x1874672572550656/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1787828650.370670:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787828650.370735:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828650/real 1787828650] req@ffff8cb992316580 x1874672572550400/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828675 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828650.370755:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb992316580 x1874672572550400/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828675 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828650.370767:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828650.370771:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787828650.370776:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb992316580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572550400:192.168.203.129@tcp:250:kworker.0 00000100:00100000:0.0:1787828650.370801:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb992314780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572550528:192.168.203.129@tcp:38:kworker.0 00000800:00000100:0.0:1787828650.370836:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787828650.370850:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828650/real 1787828650] req@ffff8cb992314780 x1874672572550528/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828675 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828650.370863:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb992314780 x1874672572550528/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828675 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828650.370871:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828650.370874:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787828650.370877:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb992314780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572550528:192.168.203.129@tcp:38:kworker.0 00000100:00100000:0.0:1787828650.370883:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb992314b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572550656:192.168.203.129@tcp:38:kworker.0 00000800:00000100:0.0:1787828650.370902:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787828650.370913:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828650/real 1787828650] req@ffff8cb992314b40 x1874672572550656/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828675 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828650.370941:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb992314b40 x1874672572550656/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828675 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828650.370949:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828650.370953:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787828650.370957:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb992314b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572550656:192.168.203.129@tcp:38:kworker.0 00010000:00010000:1.0:1787828651.202615:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787828652.226461:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787828653.250442:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787828654.274882:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787828655.298450:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787828655.490343:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828655.490354:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787828655.490358:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828655.490369:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 1607 00000100:00080000:3.0:1787828655.490382:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828655.490389:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:3.0:1787828655.490432:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb992317480 x1874672572550784/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828655.490474:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828655.490477:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787828655.490478:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828655.490481:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 1607 00000100:00080000:3.0:1787828655.490484:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828655.490486:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:3.0:1787828655.490496:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787828655.490507:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb992315e00 x1874672572550912/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828655.490514:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828655.490518:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1787828655.490520:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1787828655.490522:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb992317480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572550784:192.168.203.129@tcp:250:kworker.0 00000100:00080000:3.0:1787828655.490532:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 1607 00000100:00080000:3.0:1787828655.490536:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828655.490538:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:3.0:1787828655.490542:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787828655.490550:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb992316940 x1874672572551040/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1787828655.490600:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787828655.490642:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828655/real 1787828655] req@ffff8cb992317480 x1874672572550784/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828680 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828655.490658:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb992317480 x1874672572550784/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828680 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828655.490667:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828655.490670:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787828655.490673:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb992317480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572550784:192.168.203.129@tcp:250:kworker.0 00000100:00100000:0.0:1787828655.490699:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb992315e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572550912:192.168.203.129@tcp:38:kworker.0 00000800:00000100:0.0:1787828655.490715:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787828655.490732:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828655/real 1787828655] req@ffff8cb992315e00 x1874672572550912/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828680 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828655.490740:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb992315e00 x1874672572550912/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828680 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828655.490756:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828655.490759:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787828655.490762:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb992315e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572550912:192.168.203.129@tcp:38:kworker.0 00000100:00100000:0.0:1787828655.490766:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb992316940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572551040:192.168.203.129@tcp:38:kworker.0 00000800:00000100:0.0:1787828655.490782:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787828655.490789:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828655/real 1787828655] req@ffff8cb992316940 x1874672572551040/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828680 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828655.490804:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb992316940 x1874672572551040/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828680 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828655.490810:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828655.490812:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787828655.490814:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb992316940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572551040:192.168.203.129@tcp:38:kworker.0 00010000:00010000:1.0:1787828656.323196:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787828657.347087:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787828658.370395:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787828659.394499:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787828660.418493:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787828660.610411:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828660.610422:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787828660.610426:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828660.610437:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 1612 00000100:00080000:3.0:1787828660.610472:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828660.610481:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:3.0:1787828660.610520:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb9923161c0 x1874672572551168/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828660.610548:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828660.610551:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787828660.610552:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828660.610555:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 1612 00000100:00080000:3.0:1787828660.610557:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828660.610559:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:3.0:1787828660.610570:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787828660.610578:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb992315a40 x1874672572551296/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828660.610586:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828660.610588:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1787828660.610590:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828660.610593:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 1612 00000100:00080000:3.0:1787828660.610595:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828660.610597:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:3.0:1787828660.610601:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787828660.610603:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb9923161c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572551168:192.168.203.129@tcp:250:kworker.0 00000100:00100000:3.0:1787828660.610611:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb992314f00 x1874672572551424/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1787828660.610679:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787828660.610739:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828660/real 1787828660] req@ffff8cb9923161c0 x1874672572551168/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828685 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828660.610755:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb9923161c0 x1874672572551168/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828685 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828660.610764:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828660.610768:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787828660.610778:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb9923161c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572551168:192.168.203.129@tcp:250:kworker.0 00000100:00100000:0.0:1787828660.610784:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb992315a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572551296:192.168.203.129@tcp:38:kworker.0 00000800:00000100:0.0:1787828660.610803:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787828660.610813:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828660/real 1787828660] req@ffff8cb992315a40 x1874672572551296/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828685 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828660.610821:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb992315a40 x1874672572551296/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828685 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828660.610827:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828660.610829:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787828660.610831:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb992315a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572551296:192.168.203.129@tcp:38:kworker.0 00000100:00100000:0.0:1787828660.610858:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb992314f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572551424:192.168.203.129@tcp:38:kworker.0 00000800:00000100:0.0:1787828660.610876:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787828660.610883:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828660/real 1787828660] req@ffff8cb992314f00 x1874672572551424/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828685 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828660.610890:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb992314f00 x1874672572551424/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828685 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828660.610896:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828660.610898:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787828660.610900:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb992314f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572551424:192.168.203.129@tcp:38:kworker.0 00010000:00010000:1.0:1787828661.442454:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787828662.466412:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787828663.490432:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787828664.514409:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787828665.538451:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787828665.730360:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828665.730393:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787828665.730398:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828665.730410:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 1617 00000100:00080000:3.0:1787828665.730424:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828665.730433:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:3.0:1787828665.730478:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb9923170c0 x1874672572551552/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828665.730510:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828665.730513:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787828665.730514:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828665.730517:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 1617 00000100:00080000:3.0:1787828665.730520:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828665.730524:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:3.0:1787828665.730535:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787828665.730549:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb992314f00 x1874672572551680/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828665.730557:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828665.730560:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1787828665.730561:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828665.730566:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 1617 00000100:00080000:3.0:1787828665.730572:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1787828665.730576:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb9923170c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572551552:192.168.203.129@tcp:250:kworker.0 00000100:00080000:3.0:1787828665.730576:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:3.0:1787828665.730581:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787828665.730595:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb992315a40 x1874672572551808/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1787828665.730663:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787828665.730720:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828665/real 1787828665] req@ffff8cb9923170c0 x1874672572551552/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828690 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828665.730740:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb9923170c0 x1874672572551552/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828690 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828665.730752:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828665.730758:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787828665.730762:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb9923170c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572551552:192.168.203.129@tcp:250:kworker.0 00000100:00100000:0.0:1787828665.730790:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb992314f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572551680:192.168.203.129@tcp:38:kworker.0 00000800:00000100:0.0:1787828665.730814:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787828665.730827:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828665/real 1787828665] req@ffff8cb992314f00 x1874672572551680/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828690 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828665.730841:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb992314f00 x1874672572551680/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828690 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828665.730849:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828665.730853:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787828665.730857:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb992314f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572551680:192.168.203.129@tcp:38:kworker.0 00000100:00100000:0.0:1787828665.730864:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb992315a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572551808:192.168.203.129@tcp:38:kworker.0 00000800:00000100:0.0:1787828665.730883:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787828665.730894:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828665/real 1787828665] req@ffff8cb992315a40 x1874672572551808/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828690 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828665.730907:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb992315a40 x1874672572551808/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828690 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828665.730928:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828665.730949:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787828665.730953:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb992315a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572551808:192.168.203.129@tcp:38:kworker.0 00010000:00010000:1.0:1787828666.562535:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787828667.586375:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787828668.610414:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787828669.634458:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787828670.658442:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787828670.850344:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828670.850362:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787828670.850365:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828670.850374:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 1623 00000100:00080000:3.0:1787828670.850383:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828670.850389:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:3.0:1787828670.850422:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb9923161c0 x1874672572551936/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828670.850446:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828670.850451:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787828670.850454:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828670.850459:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 1623 00000100:00080000:3.0:1787828670.850463:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828670.850467:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:3.0:1787828670.850477:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787828670.850489:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb992316940 x1874672572552064/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787828670.850492:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb9923161c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572551936:192.168.203.129@tcp:250:kworker.0 00000100:00080000:3.0:1787828670.850501:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828670.850504:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1787828670.850506:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828670.850510:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 1623 00000100:00080000:3.0:1787828670.850513:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828670.850517:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:3.0:1787828670.850538:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787828670.850550:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb992315e00 x1874672572552192/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1787828670.850561:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787828670.850632:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828670/real 1787828670] req@ffff8cb9923161c0 x1874672572551936/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828695 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828670.850662:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb9923161c0 x1874672572551936/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828695 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828670.850674:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828670.850678:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787828670.850683:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb9923161c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572551936:192.168.203.129@tcp:250:kworker.0 00000100:00100000:0.0:1787828670.850726:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb992316940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572552064:192.168.203.129@tcp:38:kworker.0 00000800:00000100:0.0:1787828670.850751:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787828670.850765:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828670/real 1787828670] req@ffff8cb992316940 x1874672572552064/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828695 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828670.850795:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb992316940 x1874672572552064/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828695 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828670.850808:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828670.850812:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787828670.850818:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb992316940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572552064:192.168.203.129@tcp:38:kworker.0 00000100:00100000:0.0:1787828670.850838:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb992315e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572552192:192.168.203.129@tcp:38:kworker.0 00000800:00000100:0.0:1787828670.850862:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787828670.850873:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828670/real 1787828670] req@ffff8cb992315e00 x1874672572552192/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828695 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828670.850883:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb992315e00 x1874672572552192/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828695 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828670.850890:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828670.850913:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787828670.850920:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb992315e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572552192:192.168.203.129@tcp:38:kworker.0 00010000:00010000:1.0:1787828671.682488:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787828672.706417:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787828673.730420:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787828674.754426:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787828675.778407:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787828675.970367:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828675.970377:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787828675.970382:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828675.970393:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 1628 00000100:00080000:1.0:1787828675.970404:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828675.970411:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:1.0:1787828675.970454:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f38b40 x1874672572552320/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828675.970478:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828675.970481:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787828675.970482:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828675.970485:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 1628 00000100:00080000:1.0:1787828675.970505:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828675.970509:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:0.0:1787828675.970516:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f38b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572552320:192.168.203.129@tcp:250:kworker.0 00000008:00100000:1.0:1787828675.970520:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787828675.970533:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f39e00 x1874672572552448/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828675.970542:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828675.970545:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1787828675.970546:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828675.970550:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 1628 00000100:00080000:1.0:1787828675.970552:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828675.970555:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:1.0:1787828675.970558:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787828675.970567:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f3b840 x1874672572552576/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1787828675.970588:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787828675.970649:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828675/real 1787828675] req@ffff8cb999f38b40 x1874672572552320/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828700 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828675.970666:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f38b40 x1874672572552320/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828700 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828675.970675:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828675.970678:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787828675.970681:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f38b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572552320:192.168.203.129@tcp:250:kworker.0 00000100:00100000:0.0:1787828675.970707:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f39e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572552448:192.168.203.129@tcp:38:kworker.0 00000800:00000100:0.0:1787828675.970723:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787828675.970732:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828675/real 1787828675] req@ffff8cb999f39e00 x1874672572552448/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828700 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828675.970749:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f39e00 x1874672572552448/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828700 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828675.970755:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828675.970757:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787828675.970759:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f39e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572552448:192.168.203.129@tcp:38:kworker.0 00000100:00100000:0.0:1787828675.970763:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f3b840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572552576:192.168.203.129@tcp:38:kworker.0 00000800:00000100:0.0:1787828675.970786:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787828675.970793:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828675/real 1787828675] req@ffff8cb999f3b840 x1874672572552576/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828700 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828675.970800:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f3b840 x1874672572552576/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828700 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828675.970805:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828675.970806:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787828675.970808:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f3b840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572552576:192.168.203.129@tcp:38:kworker.0 00010000:00010000:1.0:1787828676.802503:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787828677.826509:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787828678.850447:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787828679.874438:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787828680.898421:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787828681.090381:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828681.090388:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787828681.090392:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828681.090400:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 1633 00000100:00080000:0.0:1787828681.090410:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828681.090415:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:0.0:1787828681.090447:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c69e00 x1874672572552704/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828681.090484:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828681.090487:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787828681.090489:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828681.090492:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 1633 00000100:00080000:0.0:1787828681.090495:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828681.090499:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:0.0:1787828681.090507:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787828681.090517:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c69e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572552704:192.168.203.129@tcp:250:kworker.0 00000100:00100000:0.0:1787828681.090518:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c6bc00 x1874672572552832/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828681.090530:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828681.090533:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1787828681.090534:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828681.090537:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 1633 00000100:00080000:0.0:1787828681.090540:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828681.090542:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:0.0:1787828681.090545:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787828681.090553:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c68f00 x1874672572552960/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1787828681.090574:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787828681.090607:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828681/real 1787828681] req@ffff8cb999c69e00 x1874672572552704/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828706 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828681.090620:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c69e00 x1874672572552704/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828706 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828681.090634:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828681.090637:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787828681.090641:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c69e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572552704:192.168.203.129@tcp:250:kworker.0 00000100:00100000:1.0:1787828681.090661:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c6bc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572552832:192.168.203.129@tcp:38:kworker.0 00000800:00000100:1.0:1787828681.090677:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787828681.090685:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828681/real 1787828681] req@ffff8cb999c6bc00 x1874672572552832/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828706 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828681.090694:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c6bc00 x1874672572552832/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828706 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828681.090699:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828681.090701:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787828681.090704:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c6bc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572552832:192.168.203.129@tcp:38:kworker.0 00000100:00100000:1.0:1787828681.090707:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c68f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572552960:192.168.203.129@tcp:38:kworker.0 00000800:00000100:1.0:1787828681.090717:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787828681.090722:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828681/real 1787828681] req@ffff8cb999c68f00 x1874672572552960/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828706 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828681.090729:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c68f00 x1874672572552960/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828706 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828681.090735:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828681.090736:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787828681.090738:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c68f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572552960:192.168.203.129@tcp:38:kworker.0 00010000:00010000:1.0:1787828681.922404:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787828682.946616:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787828683.970424:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787828684.994417:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787828686.018404:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787828686.210393:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828686.210406:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787828686.210410:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828686.210420:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 1638 00000100:00080000:2.0:1787828686.210430:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828686.210437:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:2.0:1787828686.210472:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8f840 x1874672572553088/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828686.210498:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828686.210502:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787828686.210504:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828686.210506:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 1638 00000100:00080000:2.0:1787828686.210510:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828686.210514:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:2.0:1787828686.210525:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787828686.210543:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8f840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572553088:192.168.203.129@tcp:250:kworker.0 00000100:00100000:2.0:1787828686.210545:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8ed00 x1874672572553216/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828686.210578:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828686.210582:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1787828686.210584:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828686.210587:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 1638 00000100:00080000:2.0:1787828686.210592:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828686.210596:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:2.0:1787828686.210601:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000800:00000100:1.0:1787828686.210620:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787828686.210626:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8e940 x1874672572553344/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787828686.210675:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828686/real 1787828686] req@ffff8cb998a8f840 x1874672572553088/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828711 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828686.210720:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8f840 x1874672572553088/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828711 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828686.210733:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828686.210738:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787828686.210742:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8f840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572553088:192.168.203.129@tcp:250:kworker.0 00000100:00100000:1.0:1787828686.210770:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8ed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572553216:192.168.203.129@tcp:38:kworker.0 00000800:00000100:1.0:1787828686.210826:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787828686.210869:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828686/real 1787828686] req@ffff8cb998a8ed00 x1874672572553216/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828711 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828686.210882:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8ed00 x1874672572553216/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828711 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828686.210891:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828686.210895:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787828686.210898:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8ed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572553216:192.168.203.129@tcp:38:kworker.0 00000100:00100000:1.0:1787828686.210914:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8e940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572553344:192.168.203.129@tcp:38:kworker.0 00000800:00000100:1.0:1787828686.210953:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787828686.210962:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828686/real 1787828686] req@ffff8cb998a8e940 x1874672572553344/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828711 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828686.210971:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8e940 x1874672572553344/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828711 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828686.210977:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828686.210980:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787828686.210982:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8e940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572553344:192.168.203.129@tcp:38:kworker.0 00010000:00010000:1.0:1787828687.042383:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787828688.066390:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787828689.090423:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787828690.114439:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787828691.138388:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787828691.330397:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828691.330407:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787828691.330412:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828691.330424:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 1643 00000100:00080000:0.0:1787828691.330458:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828691.330467:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:0.0:1787828691.330502:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c6b840 x1874672572553472/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828691.330548:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828691.330551:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787828691.330553:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828691.330556:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 1643 00000100:00080000:0.0:1787828691.330558:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828691.330561:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:0.0:1787828691.330570:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787828691.330579:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c69a40 x1874672572553600/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828691.330588:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828691.330591:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1787828691.330593:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828691.330598:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 1643 00000100:00100000:1.0:1787828691.330598:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c6b840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572553472:192.168.203.129@tcp:250:kworker.0 00000100:00080000:0.0:1787828691.330604:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828691.330610:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:0.0:1787828691.330615:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787828691.330641:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c68f00 x1874672572553728/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1787828691.330683:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787828691.330728:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828691/real 1787828691] req@ffff8cb999c6b840 x1874672572553472/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828716 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828691.330745:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c6b840 x1874672572553472/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828716 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828691.330769:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828691.330774:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787828691.330780:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c6b840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572553472:192.168.203.129@tcp:250:kworker.0 00000100:00100000:1.0:1787828691.330816:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c69a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572553600:192.168.203.129@tcp:38:kworker.0 00000800:00000100:1.0:1787828691.330839:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787828691.330850:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828691/real 1787828691] req@ffff8cb999c69a40 x1874672572553600/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828716 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828691.330861:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c69a40 x1874672572553600/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828716 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828691.330868:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828691.330871:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787828691.330874:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c69a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572553600:192.168.203.129@tcp:38:kworker.0 00000100:00100000:1.0:1787828691.330879:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c68f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572553728:192.168.203.129@tcp:38:kworker.0 00000800:00000100:1.0:1787828691.330891:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787828691.330914:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828691/real 1787828691] req@ffff8cb999c68f00 x1874672572553728/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828716 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828691.330922:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c68f00 x1874672572553728/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828716 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828691.330928:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828691.330930:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787828691.330932:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c68f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572553728:192.168.203.129@tcp:38:kworker.0 00010000:00010000:1.0:1787828692.162388:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787828693.186433:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787828694.210528:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787828695.234419:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787828696.258376:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787828696.450345:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828696.450373:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787828696.450377:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828696.450389:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 1648 00000100:00080000:0.0:1787828696.450400:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828696.450417:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:0.0:1787828696.450459:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c6bc00 x1874672572553856/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828696.450482:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828696.450485:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787828696.450487:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828696.450501:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 1648 00000100:00080000:0.0:1787828696.450504:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828696.450507:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:0.0:1787828696.450515:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787828696.450531:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c69e00 x1874672572553984/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828696.450540:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828696.450544:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1787828696.450548:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1787828696.450548:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c6bc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572553856:192.168.203.129@tcp:250:kworker.0 00000100:00080000:0.0:1787828696.450555:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 1648 00000100:00080000:0.0:1787828696.450559:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828696.450561:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:0.0:1787828696.450565:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787828696.450574:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c6b480 x1874672572554112/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1787828696.450639:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787828696.450675:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828696/real 1787828696] req@ffff8cb999c6bc00 x1874672572553856/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828721 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828696.450690:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c6bc00 x1874672572553856/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828721 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828696.450699:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828696.450702:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787828696.450705:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c6bc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572553856:192.168.203.129@tcp:250:kworker.0 00000100:00100000:1.0:1787828696.450740:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c69e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572553984:192.168.203.129@tcp:38:kworker.0 00000800:00000100:1.0:1787828696.450774:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787828696.450785:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828696/real 1787828696] req@ffff8cb999c69e00 x1874672572553984/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828721 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828696.450796:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c69e00 x1874672572553984/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828721 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828696.450802:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828696.450805:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787828696.450808:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c69e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572553984:192.168.203.129@tcp:38:kworker.0 00000100:00100000:1.0:1787828696.450813:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c6b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572554112:192.168.203.129@tcp:38:kworker.0 00000800:00000100:1.0:1787828696.450825:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787828696.450832:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828696/real 1787828696] req@ffff8cb999c6b480 x1874672572554112/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828721 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828696.450851:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c6b480 x1874672572554112/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828721 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828696.450858:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828696.450860:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787828696.450863:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c6b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572554112:192.168.203.129@tcp:38:kworker.0 00010000:00010000:1.0:1787828697.282457:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787828698.306410:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787828699.330447:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787828700.354486:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787828701.378599:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787828701.570440:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828701.570448:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787828701.570452:0:37:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828701.570460:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 1653 00000100:00080000:1.0:1787828701.570469:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828701.570474:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:1.0:1787828701.570505:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f39a40 x1874672572554240/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828701.570541:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828701.570545:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787828701.570551:0:37:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828701.570556:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 1653 00000100:00080000:1.0:1787828701.570561:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828701.570584:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:0.0:1787828701.570589:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f39a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572554240:192.168.203.129@tcp:250:kworker.0 00000008:00100000:1.0:1787828701.570599:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787828701.570622:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f3b840 x1874672572554368/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828701.570634:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828701.570638:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1787828701.570640:0:37:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828701.570646:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 1653 00000100:00080000:1.0:1787828701.570663:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828701.570668:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:1.0:1787828701.570673:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000800:00000100:0.0:1787828701.570678:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787828701.570705:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f39e00 x1874672572554496/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787828701.570747:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828701/real 1787828701] req@ffff8cb999f39a40 x1874672572554240/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828726 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828701.570765:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f39a40 x1874672572554240/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828726 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828701.570775:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828701.570778:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787828701.570781:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f39a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572554240:192.168.203.129@tcp:250:kworker.0 00000100:00100000:0.0:1787828701.570808:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f3b840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572554368:192.168.203.129@tcp:38:kworker.0 00000800:00000100:0.0:1787828701.570828:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787828701.570838:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828701/real 1787828701] req@ffff8cb999f3b840 x1874672572554368/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828726 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828701.570846:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f3b840 x1874672572554368/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828726 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828701.570851:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828701.570854:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787828701.570856:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f3b840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572554368:192.168.203.129@tcp:38:kworker.0 00000100:00100000:0.0:1787828701.570860:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f39e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572554496:192.168.203.129@tcp:38:kworker.0 00000800:00000100:0.0:1787828701.570872:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787828701.570891:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828701/real 1787828701] req@ffff8cb999f39e00 x1874672572554496/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828726 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828701.570902:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f39e00 x1874672572554496/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828726 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828701.570911:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828701.570915:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787828701.570919:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f39e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572554496:192.168.203.129@tcp:38:kworker.0 00010000:00010000:1.0:1787828702.402458:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787828703.426544:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787828704.450419:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787828705.474519:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787828706.498412:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787828706.690380:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828706.690391:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787828706.690395:0:37:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828706.690407:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 1658 00000100:00080000:1.0:1787828706.690418:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828706.690427:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:1.0:1787828706.690461:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f38b40 x1874672572554624/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828706.690501:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828706.690509:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787828706.690511:0:37:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828706.690515:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 1658 00000100:00080000:1.0:1787828706.690519:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828706.690524:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:1.0:1787828706.690532:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787828706.690548:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f3b480 x1874672572554752/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787828706.690549:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f38b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572554624:192.168.203.129@tcp:250:kworker.0 00000100:00080000:1.0:1787828706.690565:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828706.690568:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1787828706.690570:0:37:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828706.690574:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 1658 00000100:00080000:1.0:1787828706.690577:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828706.690580:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:1.0:1787828706.690584:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787828706.690593:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f3b0c0 x1874672572554880/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1787828706.690621:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787828706.690689:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828706/real 1787828706] req@ffff8cb999f38b40 x1874672572554624/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828731 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828706.690708:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f38b40 x1874672572554624/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828731 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828706.690717:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828706.690721:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787828706.690725:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f38b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572554624:192.168.203.129@tcp:250:kworker.0 00000100:00100000:2.0:1787828706.690746:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f3b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572554752:192.168.203.129@tcp:38:kworker.0 00000800:00000100:2.0:1787828706.690765:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787828706.690775:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828706/real 1787828706] req@ffff8cb999f3b480 x1874672572554752/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828731 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828706.690791:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f3b480 x1874672572554752/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828731 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828706.690798:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828706.690800:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787828706.690803:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f3b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572554752:192.168.203.129@tcp:38:kworker.0 00000100:00100000:2.0:1787828706.690807:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f3b0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572554880:192.168.203.129@tcp:38:kworker.0 00000800:00000100:2.0:1787828706.690824:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787828706.690832:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828706/real 1787828706] req@ffff8cb999f3b0c0 x1874672572554880/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828731 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828706.690842:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f3b0c0 x1874672572554880/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828731 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828706.690847:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828706.690850:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787828706.690852:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f3b0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572554880:192.168.203.129@tcp:38:kworker.0 00010000:00010000:1.0:1787828707.522418:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787828708.546407:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787828709.570471:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787828710.594422:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787828711.618463:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787828711.810476:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828711.810486:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787828711.810491:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828711.810502:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 1663 00000100:00080000:2.0:1787828711.810514:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828711.810522:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:2.0:1787828711.810575:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8cb40 x1874672572555008/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828711.810661:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828711.810666:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787828711.810669:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828711.810675:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 1663 00000100:00080000:2.0:1787828711.810682:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828711.810689:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:2.0:1787828711.810703:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787828711.810729:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8cb40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572555008:192.168.203.129@tcp:250:kworker.0 00000100:00100000:2.0:1787828711.810735:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8f480 x1874672572555136/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828711.810757:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828711.810762:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1787828711.810765:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828711.810771:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 1663 00000100:00080000:2.0:1787828711.810776:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828711.810783:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:2.0:1787828711.810789:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787828711.810813:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8c000 x1874672572555264/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787828711.810894:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572555136:192.168.203.129@tcp:38:kworker.0 00000100:00100000:0.0:1787828711.810925:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572555264:192.168.203.129@tcp:38:kworker.0 00010000:00010000:1.0:1787828712.642451:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787828713.666467:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787828714.690914:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:3.0:1787828714.882534:0:2381:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.129:988 00000400:00000100:3.0:1787828714.882587:0:2381:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.129@tcp at host 192.168.203.129 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:3.0:1787828714.882616:0:2381:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.129@tcp 00000800:00000100:3.0:1787828714.882659:0:2381:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.129@tcp 00000800:00000100:3.0:1787828714.882669:0:2381:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.129@tcp 00000100:00100000:0.0:1787828714.882722:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828711/real 1787828714] req@ffff8cb998a8f480 x1874672572555136/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828736 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828714.882748:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8f480 x1874672572555136/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828736 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828714.882760:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828714.882765:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787828714.882775:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572555136:192.168.203.129@tcp:38:kworker.0 00000100:00100000:0.0:1787828714.882784:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828711/real 1787828714] req@ffff8cb998a8c000 x1874672572555264/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828736 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828714.882799:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8c000 x1874672572555264/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828736 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828714.882823:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828714.882828:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787828714.882832:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572555264:192.168.203.129@tcp:38:kworker.0 00000100:00100000:0.0:1787828714.882839:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828711/real 1787828714] req@ffff8cb998a8cb40 x1874672572555008/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828736 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828714.882854:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8cb40 x1874672572555008/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828736 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828714.882862:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828714.882866:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787828714.882869:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8cb40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572555008:192.168.203.129@tcp:250:kworker.0 00010000:00010000:0.0:1787828715.714439:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787828716.738475:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787828716.930387:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828716.930398:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787828716.930403:0:37:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828716.930413:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 1669 00000100:00080000:1.0:1787828716.930424:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828716.930431:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:1.0:1787828716.930461:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f38f00 x1874672572555392/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828716.930488:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828716.930491:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787828716.930493:0:37:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828716.930497:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 1669 00000100:00080000:1.0:1787828716.930501:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828716.930505:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:1.0:1787828716.930515:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787828716.930529:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f392c0 x1874672572555520/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787828716.930543:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f38f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572555392:192.168.203.129@tcp:250:kworker.0 00000100:00080000:1.0:1787828716.930544:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828716.930554:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1787828716.930556:0:37:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828716.930561:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 1669 00000100:00080000:1.0:1787828716.930566:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828716.930602:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:1.0:1787828716.930609:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787828716.930624:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f3a580 x1874672572555648/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1787828716.930631:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787828716.930676:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828716/real 1787828716] req@ffff8cb999f38f00 x1874672572555392/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828741 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828716.930697:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f38f00 x1874672572555392/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828741 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828716.930710:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828716.930714:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787828716.930719:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f38f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572555392:192.168.203.129@tcp:250:kworker.0 00000100:00100000:0.0:1787828716.930761:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f392c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572555520:192.168.203.129@tcp:38:kworker.0 00000800:00000100:0.0:1787828716.930788:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787828716.930803:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828716/real 1787828716] req@ffff8cb999f392c0 x1874672572555520/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828741 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828716.930817:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f392c0 x1874672572555520/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828741 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828716.930825:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828716.930829:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787828716.930832:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f392c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572555520:192.168.203.129@tcp:38:kworker.0 00000100:00100000:0.0:1787828716.930839:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f3a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572555648:192.168.203.129@tcp:38:kworker.0 00000800:00000100:0.0:1787828716.930861:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787828716.930874:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828716/real 1787828716] req@ffff8cb999f3a580 x1874672572555648/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828741 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828716.930887:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f3a580 x1874672572555648/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828741 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828716.930896:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828716.930899:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787828716.930903:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f3a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572555648:192.168.203.129@tcp:38:kworker.0 00010000:00010000:0.0:1787828717.762638:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787828718.786660:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787828719.810802:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787828720.834741:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787828721.859080:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787828722.050364:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828722.050375:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787828722.050380:0:37:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828722.050391:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 1674 00000100:00080000:1.0:1787828722.050403:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828722.050409:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:1.0:1787828722.050453:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f383c0 x1874672572555776/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828722.050479:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828722.050482:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787828722.050483:0:37:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828722.050486:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 1674 00000100:00080000:1.0:1787828722.050488:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828722.050491:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:1.0:1787828722.050502:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787828722.050515:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f3a580 x1874672572555904/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828722.050528:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1787828722.050530:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f383c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572555776:192.168.203.129@tcp:250:kworker.0 00000100:00080000:1.0:1787828722.050572:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1787828722.050577:0:37:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828722.050586:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 1674 00000100:00080000:1.0:1787828722.050592:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828722.050597:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:1.0:1787828722.050603:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787828722.050618:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f392c0 x1874672572556032/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1787828722.050632:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787828722.050679:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828722/real 1787828722] req@ffff8cb999f383c0 x1874672572555776/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828747 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828722.050700:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f383c0 x1874672572555776/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828747 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828722.050712:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828722.050716:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787828722.050721:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f383c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572555776:192.168.203.129@tcp:250:kworker.0 00000100:00100000:0.0:1787828722.050750:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f3a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572555904:192.168.203.129@tcp:38:kworker.0 00000800:00000100:0.0:1787828722.050769:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787828722.050779:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828722/real 1787828722] req@ffff8cb999f3a580 x1874672572555904/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828747 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828722.050787:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f3a580 x1874672572555904/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828747 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828722.050801:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828722.050803:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787828722.050805:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f3a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572555904:192.168.203.129@tcp:38:kworker.0 00000100:00100000:0.0:1787828722.050810:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f392c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572556032:192.168.203.129@tcp:38:kworker.0 00000800:00000100:0.0:1787828722.050822:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787828722.050829:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828722/real 1787828722] req@ffff8cb999f392c0 x1874672572556032/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828747 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828722.050848:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f392c0 x1874672572556032/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828747 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828722.050853:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828722.050856:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787828722.050858:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f392c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572556032:192.168.203.129@tcp:38:kworker.0 00010000:00010000:2.0:1787828722.882653:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787828723.907399:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787828724.930487:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787828725.954384:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787828726.978414:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787828727.170312:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828727.170321:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787828727.170324:0:37:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828727.170332:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 1679 00000100:00080000:1.0:1787828727.170341:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828727.170346:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:1.0:1787828727.170381:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f38f00 x1874672572556160/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828727.170410:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828727.170414:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787828727.170416:0:37:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828727.170419:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 1679 00000100:00080000:1.0:1787828727.170422:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828727.170425:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:1.0:1787828727.170446:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787828727.170459:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f3b0c0 x1874672572556288/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787828727.170461:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f38f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572556160:192.168.203.129@tcp:250:kworker.0 00000100:00080000:1.0:1787828727.170476:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828727.170479:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1787828727.170480:0:37:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828727.170484:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 1679 00000100:00080000:1.0:1787828727.170487:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828727.170489:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:1.0:1787828727.170493:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787828727.170502:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f3b480 x1874672572556416/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1787828727.170556:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787828727.170593:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828727/real 1787828727] req@ffff8cb999f38f00 x1874672572556160/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828752 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828727.170608:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f38f00 x1874672572556160/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828752 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828727.170617:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828727.170620:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787828727.170624:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f38f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572556160:192.168.203.129@tcp:250:kworker.0 00000100:00100000:0.0:1787828727.170646:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f3b0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572556288:192.168.203.129@tcp:38:kworker.0 00000800:00000100:0.0:1787828727.170663:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787828727.170672:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828727/real 1787828727] req@ffff8cb999f3b0c0 x1874672572556288/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828752 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828727.170681:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f3b0c0 x1874672572556288/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828752 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828727.170714:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828727.170717:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787828727.170720:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f3b0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572556288:192.168.203.129@tcp:38:kworker.0 00000100:00100000:0.0:1787828727.170724:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f3b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572556416:192.168.203.129@tcp:38:kworker.0 00000800:00000100:0.0:1787828727.170738:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787828727.170744:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828727/real 1787828727] req@ffff8cb999f3b480 x1874672572556416/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828752 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828727.170752:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f3b480 x1874672572556416/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828752 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828727.170757:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828727.170759:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787828727.170761:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f3b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572556416:192.168.203.129@tcp:38:kworker.0 00010000:00010000:0.0:1787828728.002419:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787828729.026399:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787828730.050372:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787828731.074434:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787828732.098409:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787828732.290394:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828732.290403:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787828732.290407:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828732.290417:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 1684 00000100:00080000:0.0:1787828732.290427:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828732.290435:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:0.0:1787828732.290498:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c692c0 x1874672572556544/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828732.290543:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828732.290547:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787828732.290550:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828732.290554:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 1684 00000100:00080000:0.0:1787828732.290560:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828732.290566:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:0.0:1787828732.290577:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787828732.290611:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c692c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572556544:192.168.203.129@tcp:250:kworker.0 00000100:00100000:0.0:1787828732.290616:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c69680 x1874672572556672/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828732.290629:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828732.290633:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1787828732.290635:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828732.290639:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 1684 00000100:00080000:0.0:1787828732.290644:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828732.290648:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:0.0:1787828732.290653:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787828732.290666:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c68000 x1874672572556800/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1787828732.290677:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787828732.290725:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828732/real 1787828732] req@ffff8cb999c692c0 x1874672572556544/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828757 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828732.290741:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c692c0 x1874672572556544/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828757 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828732.290750:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828732.290754:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787828732.290757:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c692c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572556544:192.168.203.129@tcp:250:kworker.0 00000100:00100000:2.0:1787828732.290774:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c69680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572556672:192.168.203.129@tcp:38:kworker.0 00000800:00000100:2.0:1787828732.290789:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787828732.290798:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828732/real 1787828732] req@ffff8cb999c69680 x1874672572556672/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828757 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828732.290806:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c69680 x1874672572556672/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828757 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828732.290812:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828732.290814:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787828732.290816:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c69680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572556672:192.168.203.129@tcp:38:kworker.0 00000100:00100000:2.0:1787828732.290820:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c68000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572556800:192.168.203.129@tcp:38:kworker.0 00000800:00000100:2.0:1787828732.290830:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787828732.290835:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828732/real 1787828732] req@ffff8cb999c68000 x1874672572556800/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828757 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828732.290842:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c68000 x1874672572556800/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828757 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828732.290857:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828732.290860:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787828732.290862:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c68000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572556800:192.168.203.129@tcp:38:kworker.0 00010000:00010000:1.0:1787828733.122422:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787828734.146410:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787828735.170400:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787828736.194457:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787828737.218496:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787828737.410371:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828737.410380:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787828737.410384:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828737.410392:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 1689 00000100:00080000:2.0:1787828737.410402:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828737.410408:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:2.0:1787828737.410452:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8d680 x1874672572556928/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828737.410480:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828737.410483:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787828737.410484:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828737.410487:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 1689 00000100:00080000:2.0:1787828737.410491:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828737.410495:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:2.0:1787828737.410503:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787828737.410513:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8e580 x1874672572557056/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787828737.410528:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8d680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572556928:192.168.203.129@tcp:250:kworker.0 00000100:00080000:2.0:1787828737.410530:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828737.410535:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1787828737.410556:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828737.410565:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 1689 00000100:00080000:2.0:1787828737.410570:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828737.410574:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:2.0:1787828737.410579:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787828737.410590:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8c780 x1874672572557184/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1787828737.410644:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787828737.410710:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828737/real 1787828737] req@ffff8cb998a8d680 x1874672572556928/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828762 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828737.410726:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8d680 x1874672572556928/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828762 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828737.410736:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828737.410740:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787828737.410743:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8d680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572556928:192.168.203.129@tcp:250:kworker.0 00000100:00100000:0.0:1787828737.410770:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8e580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572557056:192.168.203.129@tcp:38:kworker.0 00000800:00000100:0.0:1787828737.410790:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787828737.410800:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828737/real 1787828737] req@ffff8cb998a8e580 x1874672572557056/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828762 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828737.410809:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8e580 x1874672572557056/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828762 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828737.410815:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828737.410817:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787828737.410820:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8e580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572557056:192.168.203.129@tcp:38:kworker.0 00000100:00100000:0.0:1787828737.410824:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8c780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572557184:192.168.203.129@tcp:38:kworker.0 00000800:00000100:0.0:1787828737.410836:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787828737.410843:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828737/real 1787828737] req@ffff8cb998a8c780 x1874672572557184/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828762 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828737.410864:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8c780 x1874672572557184/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828762 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828737.410870:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828737.410872:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787828737.410874:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8c780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572557184:192.168.203.129@tcp:38:kworker.0 00010000:00010000:0.0:1787828738.242487:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787828739.266526:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787828740.290561:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787828741.314438:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787828742.338613:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787828742.530470:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828742.530482:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787828742.530490:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828742.530501:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 1694 00000100:00080000:0.0:1787828742.530515:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828742.530525:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:0.0:1787828742.530565:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c6a1c0 x1874672572557312/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828742.530602:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828742.530607:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787828742.530610:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828742.530614:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 1694 00000100:00080000:0.0:1787828742.530617:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828742.530621:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:0.0:1787828742.530631:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787828742.530660:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c6a580 x1874672572557440/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828742.530675:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828742.530680:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1787828742.530686:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1787828742.530687:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c6a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572557312:192.168.203.129@tcp:250:kworker.0 00000100:00080000:0.0:1787828742.530697:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 1694 00000100:00080000:0.0:1787828742.530707:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828742.530712:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:0.0:1787828742.530720:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787828742.530737:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c68000 x1874672572557568/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1787828742.530789:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787828742.530861:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828742/real 1787828742] req@ffff8cb999c6a1c0 x1874672572557312/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828767 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828742.530895:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c6a1c0 x1874672572557312/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828767 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828742.530913:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828742.530921:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787828742.530928:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c6a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572557312:192.168.203.129@tcp:250:kworker.0 00000100:00100000:2.0:1787828742.530965:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c6a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572557440:192.168.203.129@tcp:38:kworker.0 00000800:00000100:2.0:1787828742.531006:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787828742.531031:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828742/real 1787828742] req@ffff8cb999c6a580 x1874672572557440/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828767 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828742.531056:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c6a580 x1874672572557440/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828767 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828742.531095:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828742.531104:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787828742.531110:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c6a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572557440:192.168.203.129@tcp:38:kworker.0 00000100:00100000:2.0:1787828742.531122:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c68000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572557568:192.168.203.129@tcp:38:kworker.0 00000800:00000100:2.0:1787828742.531168:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787828742.531195:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828742/real 1787828742] req@ffff8cb999c68000 x1874672572557568/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828767 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828742.531237:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c68000 x1874672572557568/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828767 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828742.531298:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828742.531307:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787828742.531315:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c68000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572557568:192.168.203.129@tcp:38:kworker.0 00010000:00010000:0.0:1787828743.362418:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787828744.386449:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787828745.410456:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787828746.434526:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787828747.458428:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787828747.650412:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828747.650423:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787828747.650427:0:37:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828747.650437:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 1699 00000100:00080000:1.0:1787828747.650448:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828747.650455:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:1.0:1787828747.650494:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f38b40 x1874672572557696/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828747.650531:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828747.650534:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787828747.650535:0:37:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828747.650539:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 1699 00000100:00080000:1.0:1787828747.650542:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828747.650545:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:1.0:1787828747.650554:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787828747.650565:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f39e00 x1874672572557824/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828747.650573:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1787828747.650575:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f38b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572557696:192.168.203.129@tcp:250:kworker.0 00000100:00080000:1.0:1787828747.650580:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1787828747.650583:0:37:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828747.650587:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 1699 00000100:00080000:1.0:1787828747.650589:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828747.650591:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:1.0:1787828747.650595:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787828747.650603:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f3b840 x1874672572557952/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1787828747.650651:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787828747.650689:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828747/real 1787828747] req@ffff8cb999f38b40 x1874672572557696/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828772 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828747.650705:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f38b40 x1874672572557696/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828772 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828747.650714:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828747.650718:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787828747.650728:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f38b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572557696:192.168.203.129@tcp:250:kworker.0 00000100:00100000:2.0:1787828747.650766:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f39e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572557824:192.168.203.129@tcp:38:kworker.0 00000800:00000100:2.0:1787828747.650783:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787828747.650792:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828747/real 1787828747] req@ffff8cb999f39e00 x1874672572557824/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828772 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828747.650800:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f39e00 x1874672572557824/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828772 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828747.650806:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828747.650808:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787828747.650810:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f39e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572557824:192.168.203.129@tcp:38:kworker.0 00000100:00100000:2.0:1787828747.650814:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f3b840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572557952:192.168.203.129@tcp:38:kworker.0 00000800:00000100:2.0:1787828747.650824:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787828747.650830:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828747/real 1787828747] req@ffff8cb999f3b840 x1874672572557952/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828772 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828747.650837:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f3b840 x1874672572557952/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828772 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828747.650842:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828747.650844:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787828747.650845:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f3b840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572557952:192.168.203.129@tcp:38:kworker.0 00010000:00010000:1.0:1787828748.482458:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787828749.506514:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787828750.530379:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787828751.554400:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787828752.578506:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787828752.770490:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828752.770500:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787828752.770521:0:37:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828752.770531:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 1704 00000100:00080000:2.0:1787828752.770541:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828752.770549:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:2.0:1787828752.770581:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8cf00 x1874672572558080/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828752.770617:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828752.770620:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787828752.770622:0:37:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828752.770626:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 1704 00000100:00080000:2.0:1787828752.770629:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828752.770633:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:2.0:1787828752.770642:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787828752.770652:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8f0c0 x1874672572558208/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828752.770663:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828752.770666:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1787828752.770669:0:37:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1787828752.770673:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8cf00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572558080:192.168.203.129@tcp:250:kworker.0 00000100:00080000:2.0:1787828752.770676:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 1704 00000100:00080000:2.0:1787828752.770684:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828752.770687:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:2.0:1787828752.770691:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787828752.770704:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8fc00 x1874672572558336/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1787828752.770769:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787828752.770819:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828752/real 1787828752] req@ffff8cb998a8cf00 x1874672572558080/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828777 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828752.770841:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8cf00 x1874672572558080/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828777 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828752.770853:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828752.770866:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787828752.770870:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8cf00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572558080:192.168.203.129@tcp:250:kworker.0 00000100:00100000:0.0:1787828752.770896:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8f0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572558208:192.168.203.129@tcp:38:kworker.0 00000800:00000100:0.0:1787828752.770920:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787828752.770933:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828752/real 1787828752] req@ffff8cb998a8f0c0 x1874672572558208/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828777 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828752.770944:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8f0c0 x1874672572558208/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828777 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828752.770952:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828752.770956:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787828752.770959:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8f0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572558208:192.168.203.129@tcp:38:kworker.0 00000100:00100000:0.0:1787828752.770964:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8fc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572558336:192.168.203.129@tcp:38:kworker.0 00000800:00000100:0.0:1787828752.770978:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787828752.770986:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828752/real 1787828752] req@ffff8cb998a8fc00 x1874672572558336/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828777 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828752.770997:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8fc00 x1874672572558336/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828777 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828752.771004:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828752.771006:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787828752.771009:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8fc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572558336:192.168.203.129@tcp:38:kworker.0 00010000:00010000:0.0:1787828753.602472:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787828754.626441:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787828755.650417:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787828756.674387:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787828757.698490:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787828757.890419:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828757.890431:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787828757.890436:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828757.890452:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 1710 00000100:00080000:2.0:1787828757.890466:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828757.890473:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:2.0:1787828757.890518:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8e1c0 x1874672572558464/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828757.890547:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828757.890550:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787828757.890551:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828757.890553:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 1710 00000100:00080000:2.0:1787828757.890556:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828757.890558:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:2.0:1787828757.890568:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787828757.890577:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8da40 x1874672572558592/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828757.890584:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828757.890586:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1787828757.890588:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828757.890591:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 1710 00000100:00080000:2.0:1787828757.890593:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828757.890595:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:2.0:1787828757.890617:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787828757.890620:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8e1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572558464:192.168.203.129@tcp:250:kworker.0 00000100:00100000:2.0:1787828757.890627:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8de00 x1874672572558720/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1787828757.890701:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787828757.890758:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828757/real 1787828757] req@ffff8cb998a8e1c0 x1874672572558464/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828782 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828757.890775:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8e1c0 x1874672572558464/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828782 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828757.890785:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828757.890788:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787828757.890799:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8e1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572558464:192.168.203.129@tcp:250:kworker.0 00000100:00100000:0.0:1787828757.890807:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8da40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572558592:192.168.203.129@tcp:38:kworker.0 00000800:00000100:0.0:1787828757.890827:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787828757.890836:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828757/real 1787828757] req@ffff8cb998a8da40 x1874672572558592/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828782 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828757.890844:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8da40 x1874672572558592/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828782 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828757.890851:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828757.890853:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787828757.890855:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8da40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572558592:192.168.203.129@tcp:38:kworker.0 00000100:00100000:0.0:1787828757.890880:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8de00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572558720:192.168.203.129@tcp:38:kworker.0 00000800:00000100:0.0:1787828757.890901:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787828757.890909:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828757/real 1787828757] req@ffff8cb998a8de00 x1874672572558720/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828782 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828757.890916:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8de00 x1874672572558720/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828782 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828757.890922:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828757.890923:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787828757.890925:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8de00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572558720:192.168.203.129@tcp:38:kworker.0 00010000:00010000:1.0:1787828758.722566:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787828759.746460:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787828760.770455:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787828761.794446:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787828762.818554:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787828763.010415:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828763.010428:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787828763.010433:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828763.010456:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 1715 00000100:00080000:2.0:1787828763.010469:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828763.010476:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:2.0:1787828763.010525:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8c3c0 x1874672572558848/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828763.010553:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828763.010557:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787828763.010558:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828763.010561:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 1715 00000100:00080000:2.0:1787828763.010566:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828763.010569:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:2.0:1787828763.010589:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787828763.010600:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8d2c0 x1874672572558976/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828763.010611:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828763.010615:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1787828763.010633:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1787828763.010634:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8c3c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572558848:192.168.203.129@tcp:250:kworker.0 00000100:00080000:2.0:1787828763.010650:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 1715 00000100:00080000:2.0:1787828763.010659:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828763.010667:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:2.0:1787828763.010673:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787828763.010686:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8de00 x1874672572559104/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1787828763.010773:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787828763.010844:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828763/real 1787828763] req@ffff8cb998a8c3c0 x1874672572558848/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828788 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828763.010887:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8c3c0 x1874672572558848/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828788 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828763.010899:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828763.010902:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787828763.010906:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8c3c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572558848:192.168.203.129@tcp:250:kworker.0 00000100:00100000:0.0:1787828763.010933:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8d2c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572558976:192.168.203.129@tcp:38:kworker.0 00000800:00000100:0.0:1787828763.010995:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787828763.011018:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828763/real 1787828763] req@ffff8cb998a8d2c0 x1874672572558976/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828788 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828763.011077:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8d2c0 x1874672572558976/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828788 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828763.011084:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828763.011086:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787828763.011089:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8d2c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572558976:192.168.203.129@tcp:38:kworker.0 00000100:00100000:0.0:1787828763.011094:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8de00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572559104:192.168.203.129@tcp:38:kworker.0 00000800:00000100:0.0:1787828763.011111:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787828763.011138:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828763/real 1787828763] req@ffff8cb998a8de00 x1874672572559104/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828788 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828763.011152:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8de00 x1874672572559104/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828788 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828763.011161:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828763.011164:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787828763.011168:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8de00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572559104:192.168.203.129@tcp:38:kworker.0 00010000:00010000:1.0:1787828763.842561:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787828764.866454:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787828765.890516:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787828766.914445:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787828767.938434:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787828768.130348:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828768.130358:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787828768.130362:0:37:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828768.130372:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 1720 00000100:00080000:0.0:1787828768.130382:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828768.130390:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:0.0:1787828768.130425:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c69680 x1874672572559232/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828768.130467:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828768.130471:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787828768.130474:0:37:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828768.130478:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 1720 00000100:00080000:0.0:1787828768.130483:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828768.130495:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:2.0:1787828768.130505:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c69680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572559232:192.168.203.129@tcp:250:kworker.0 00000008:00100000:0.0:1787828768.130506:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787828768.130527:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c692c0 x1874672572559360/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828768.130539:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828768.130542:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1787828768.130544:0:37:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828768.130547:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 1720 00000100:00080000:0.0:1787828768.130553:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828768.130557:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000800:00000100:2.0:1787828768.130562:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000008:00100000:0.0:1787828768.130563:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787828768.130578:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c6b480 x1874672572559488/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787828768.130614:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828768/real 1787828768] req@ffff8cb999c69680 x1874672572559232/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828793 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828768.130635:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c69680 x1874672572559232/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828793 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828768.130645:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828768.130649:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787828768.130652:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c69680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572559232:192.168.203.129@tcp:250:kworker.0 00000100:00100000:2.0:1787828768.130671:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c692c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572559360:192.168.203.129@tcp:38:kworker.0 00000800:00000100:2.0:1787828768.130689:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787828768.130698:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828768/real 1787828768] req@ffff8cb999c692c0 x1874672572559360/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828793 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828768.130707:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c692c0 x1874672572559360/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828793 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828768.130723:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828768.130725:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787828768.130728:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c692c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572559360:192.168.203.129@tcp:38:kworker.0 00000100:00100000:2.0:1787828768.130732:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c6b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572559488:192.168.203.129@tcp:38:kworker.0 00000800:00000100:2.0:1787828768.130746:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787828768.130754:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828768/real 1787828768] req@ffff8cb999c6b480 x1874672572559488/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828793 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828768.130761:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c6b480 x1874672572559488/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828793 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828768.130768:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828768.130771:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787828768.130774:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c6b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572559488:192.168.203.129@tcp:38:kworker.0 00010000:00010000:2.0:1787828768.962381:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787828769.986393:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787828771.010423:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787828772.034458:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787828773.058398:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787828773.250387:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828773.250399:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787828773.250404:0:37:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828773.250414:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 1725 00000100:00080000:2.0:1787828773.250424:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828773.250429:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:2.0:1787828773.250471:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8da40 x1874672572559616/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828773.250511:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828773.250515:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787828773.250516:0:37:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828773.250519:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 1725 00000100:00080000:2.0:1787828773.250523:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828773.250526:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:2.0:1787828773.250535:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787828773.250542:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8da40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572559616:192.168.203.129@tcp:250:kworker.0 00000100:00100000:2.0:1787828773.250548:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8e1c0 x1874672572559744/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828773.250557:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828773.250560:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1787828773.250562:0:37:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828773.250564:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 1725 00000100:00080000:2.0:1787828773.250567:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828773.250569:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:2.0:1787828773.250572:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787828773.250580:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8fc00 x1874672572559872/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1787828773.250605:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787828773.250647:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828773/real 1787828773] req@ffff8cb998a8da40 x1874672572559616/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828798 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828773.250682:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8da40 x1874672572559616/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828798 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828773.250691:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828773.250694:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787828773.250697:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8da40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572559616:192.168.203.129@tcp:250:kworker.0 00000100:00100000:0.0:1787828773.250717:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8e1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572559744:192.168.203.129@tcp:38:kworker.0 00000800:00000100:0.0:1787828773.250736:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787828773.250745:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828773/real 1787828773] req@ffff8cb998a8e1c0 x1874672572559744/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828798 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828773.250754:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8e1c0 x1874672572559744/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828798 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828773.250760:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828773.250764:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787828773.250767:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8e1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572559744:192.168.203.129@tcp:38:kworker.0 00000100:00100000:0.0:1787828773.250774:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8fc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572559872:192.168.203.129@tcp:38:kworker.0 00000800:00000100:0.0:1787828773.250802:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787828773.250815:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828773/real 1787828773] req@ffff8cb998a8fc00 x1874672572559872/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828798 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828773.250825:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8fc00 x1874672572559872/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828798 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828773.250851:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828773.250854:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787828773.250857:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8fc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572559872:192.168.203.129@tcp:38:kworker.0 00010000:00010000:1.0:1787828774.082395:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787828775.106423:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787828776.130513:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787828777.154446:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787828778.178427:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787828778.370329:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828778.370337:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787828778.370352:0:37:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828778.370374:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 1730 00000100:00080000:2.0:1787828778.370383:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828778.370389:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:2.0:1787828778.370416:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8f0c0 x1874672572560000/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828778.370439:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828778.370442:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787828778.370443:0:37:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828778.370446:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 1730 00000100:00080000:2.0:1787828778.370448:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828778.370450:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:2.0:1787828778.370458:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787828778.370466:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8cf00 x1874672572560128/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828778.370474:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828778.370476:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1787828778.370479:0:37:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1787828778.370481:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8f0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572560000:192.168.203.129@tcp:250:kworker.0 00000100:00080000:2.0:1787828778.370484:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 1730 00000100:00080000:2.0:1787828778.370489:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828778.370492:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:2.0:1787828778.370496:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787828778.370506:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8c780 x1874672572560256/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787828778.370617:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8cf00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572560128:192.168.203.129@tcp:38:kworker.0 00000100:00100000:0.0:1787828778.370657:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8c780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572560256:192.168.203.129@tcp:38:kworker.0 00010000:00010000:1.0:1787828779.202419:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787828780.226486:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787828781.250692:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:0.0:1787828781.442740:0:2382:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.129:988 00000400:00000100:0.0:1787828781.442797:0:2382:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.129@tcp at host 192.168.203.129 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:0.0:1787828781.442819:0:2382:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.129@tcp 00000800:00000100:0.0:1787828781.442876:0:2382:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.129@tcp 00000800:00000100:0.0:1787828781.442887:0:2382:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.129@tcp 00000100:00100000:2.0:1787828781.442956:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828778/real 1787828781] req@ffff8cb998a8cf00 x1874672572560128/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828803 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828781.442995:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8cf00 x1874672572560128/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828803 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828781.443009:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828781.443016:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787828781.443022:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8cf00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572560128:192.168.203.129@tcp:38:kworker.0 00000100:00100000:2.0:1787828781.443032:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828778/real 1787828781] req@ffff8cb998a8c780 x1874672572560256/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828803 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828781.443046:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8c780 x1874672572560256/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828803 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828781.443054:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828781.443059:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787828781.443062:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8c780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572560256:192.168.203.129@tcp:38:kworker.0 00000100:00100000:2.0:1787828781.443070:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828778/real 1787828781] req@ffff8cb998a8f0c0 x1874672572560000/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828803 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828781.443084:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8f0c0 x1874672572560000/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828803 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828781.443092:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828781.443096:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787828781.443120:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8f0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572560000:192.168.203.129@tcp:250:kworker.0 00010000:00010000:1.0:1787828782.274456:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787828783.298715:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787828783.490427:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828783.490437:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787828783.490441:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828783.490452:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 1735 00000100:00080000:1.0:1787828783.490464:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828783.490470:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:1.0:1787828783.490532:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f39a40 x1874672572560384/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828783.490582:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828783.490586:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787828783.490588:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828783.490592:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 1735 00000100:00080000:1.0:1787828783.490597:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828783.490602:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:1.0:1787828783.490635:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787828783.490644:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f39a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572560384:192.168.203.129@tcp:250:kworker.0 00000100:00100000:1.0:1787828783.490655:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f3b840 x1874672572560512/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828783.490668:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828783.490671:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1787828783.490672:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828783.490675:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 1735 00000100:00080000:1.0:1787828783.490679:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828783.490682:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:1.0:1787828783.490686:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787828783.490697:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f39e00 x1874672572560640/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1787828783.490739:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787828783.490798:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828783/real 1787828783] req@ffff8cb999f39a40 x1874672572560384/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828808 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828783.490822:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f39a40 x1874672572560384/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828808 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828783.490835:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828783.490840:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787828783.490845:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f39a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572560384:192.168.203.129@tcp:250:kworker.0 00000100:00100000:2.0:1787828783.490879:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f3b840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572560512:192.168.203.129@tcp:38:kworker.0 00000800:00000100:2.0:1787828783.490908:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787828783.490925:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828783/real 1787828783] req@ffff8cb999f3b840 x1874672572560512/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828808 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828783.490939:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f3b840 x1874672572560512/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828808 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828783.490948:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828783.490951:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787828783.490955:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f3b840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572560512:192.168.203.129@tcp:38:kworker.0 00000100:00100000:2.0:1787828783.490962:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f39e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572560640:192.168.203.129@tcp:38:kworker.0 00000800:00000100:2.0:1787828783.490984:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787828783.491008:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828783/real 1787828783] req@ffff8cb999f39e00 x1874672572560640/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828808 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828783.491022:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f39e00 x1874672572560640/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828808 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828783.491031:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828783.491034:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787828783.491038:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f39e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572560640:192.168.203.129@tcp:38:kworker.0 00010000:00010000:2.0:1787828784.322816:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787828785.346520:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787828786.370496:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787828787.395442:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787828788.418683:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787828788.610341:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828788.610349:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787828788.610353:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828788.610361:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 1740 00000100:00080000:1.0:1787828788.610370:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828788.610375:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:1.0:1787828788.610406:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f38b40 x1874672572560768/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828788.610429:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828788.610431:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787828788.610446:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828788.610450:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 1740 00000100:00080000:1.0:1787828788.610453:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828788.610457:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:1.0:1787828788.610466:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787828788.610480:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f38b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572560768:192.168.203.129@tcp:250:kworker.0 00000100:00100000:1.0:1787828788.610487:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f3b480 x1874672572560896/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828788.610497:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828788.610499:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1787828788.610501:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828788.610504:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 1740 00000100:00080000:1.0:1787828788.610506:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828788.610509:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:1.0:1787828788.610516:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787828788.610524:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f3b0c0 x1874672572561024/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1787828788.610545:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787828788.610594:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828788/real 1787828788] req@ffff8cb999f38b40 x1874672572560768/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828813 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828788.610610:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f38b40 x1874672572560768/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828813 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828788.610618:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828788.610621:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787828788.610624:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f38b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572560768:192.168.203.129@tcp:250:kworker.0 00000100:00100000:2.0:1787828788.610644:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f3b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572560896:192.168.203.129@tcp:38:kworker.0 00000800:00000100:2.0:1787828788.610661:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787828788.610670:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828788/real 1787828788] req@ffff8cb999f3b480 x1874672572560896/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828813 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828788.610685:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f3b480 x1874672572560896/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828813 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828788.610691:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828788.610693:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787828788.610695:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f3b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572560896:192.168.203.129@tcp:38:kworker.0 00000100:00100000:2.0:1787828788.610699:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f3b0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572561024:192.168.203.129@tcp:38:kworker.0 00000800:00000100:2.0:1787828788.610715:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787828788.610726:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828788/real 1787828788] req@ffff8cb999f3b0c0 x1874672572561024/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828813 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828788.610739:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f3b0c0 x1874672572561024/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828813 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828788.610747:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828788.610751:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787828788.610755:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f3b0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572561024:192.168.203.129@tcp:38:kworker.0 00010000:00010000:0.0:1787828789.442880:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787828790.466992:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787828791.490382:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787828792.514421:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787828793.538411:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787828793.730329:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828793.730337:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787828793.730341:0:37:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828793.730348:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 1745 00000100:00080000:2.0:1787828793.730357:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828793.730375:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:2.0:1787828793.730401:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8f0c0 x1874672572561152/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828793.730426:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828793.730429:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787828793.730430:0:37:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828793.730433:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 1745 00000100:00080000:2.0:1787828793.730436:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828793.730438:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:2.0:1787828793.730446:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787828793.730459:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8c780 x1874672572561280/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828793.730468:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1787828793.730471:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8f0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572561152:192.168.203.129@tcp:250:kworker.0 00000100:00080000:2.0:1787828793.730471:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1787828793.730475:0:37:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828793.730482:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 1745 00000100:00080000:2.0:1787828793.730484:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828793.730486:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:2.0:1787828793.730490:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787828793.730499:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8cf00 x1874672572561408/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1787828793.730542:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787828793.730592:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828793/real 1787828793] req@ffff8cb998a8f0c0 x1874672572561152/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828818 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828793.730612:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8f0c0 x1874672572561152/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828818 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828793.730623:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828793.730628:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787828793.730632:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8f0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572561152:192.168.203.129@tcp:250:kworker.0 00000100:00100000:0.0:1787828793.730662:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8c780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572561280:192.168.203.129@tcp:38:kworker.0 00000800:00000100:0.0:1787828793.730698:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787828793.730712:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828793/real 1787828793] req@ffff8cb998a8c780 x1874672572561280/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828818 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828793.730726:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8c780 x1874672572561280/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828818 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828793.730735:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828793.730739:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787828793.730743:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8c780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572561280:192.168.203.129@tcp:38:kworker.0 00000100:00100000:0.0:1787828793.730750:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8cf00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572561408:192.168.203.129@tcp:38:kworker.0 00000800:00000100:0.0:1787828793.730781:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787828793.730794:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828793/real 1787828793] req@ffff8cb998a8cf00 x1874672572561408/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828818 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828793.730807:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8cf00 x1874672572561408/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828818 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828793.730816:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828793.730820:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787828793.730823:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8cf00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572561408:192.168.203.129@tcp:38:kworker.0 00010000:00010000:2.0:1787828794.562429:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787828795.586413:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787828796.610458:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787828797.634394:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787828798.658422:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787828798.850382:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828798.850390:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787828798.850409:0:37:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828798.850435:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 1751 00000100:00080000:0.0:1787828798.850445:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828798.850450:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:0.0:1787828798.850513:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c69e00 x1874672572561536/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828798.850534:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828798.850537:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787828798.850539:0:37:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828798.850542:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 1751 00000100:00080000:0.0:1787828798.850544:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828798.850547:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:0.0:1787828798.850554:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787828798.850563:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c6bc00 x1874672572561664/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828798.850571:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828798.850575:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00100000:2.0:1787828798.850576:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c69e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572561536:192.168.203.129@tcp:250:kworker.0 00000100:00080000:0.0:1787828798.850577:0:37:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828798.850585:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 1751 00000100:00080000:0.0:1787828798.850588:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828798.850590:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:0.0:1787828798.850593:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787828798.850602:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c68f00 x1874672572561792/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1787828798.850647:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787828798.850694:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828798/real 1787828798] req@ffff8cb999c69e00 x1874672572561536/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828823 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828798.850713:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c69e00 x1874672572561536/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828823 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828798.850725:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828798.850729:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787828798.850734:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c69e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572561536:192.168.203.129@tcp:250:kworker.0 00000100:00100000:2.0:1787828798.850758:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c6bc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572561664:192.168.203.129@tcp:38:kworker.0 00000800:00000100:2.0:1787828798.850778:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787828798.850788:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828798/real 1787828798] req@ffff8cb999c6bc00 x1874672572561664/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828823 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828798.850805:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c6bc00 x1874672572561664/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828823 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828798.850810:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828798.850813:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787828798.850815:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c6bc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572561664:192.168.203.129@tcp:38:kworker.0 00000100:00100000:2.0:1787828798.850819:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c68f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572561792:192.168.203.129@tcp:38:kworker.0 00000800:00000100:2.0:1787828798.850831:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787828798.850838:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828798/real 1787828798] req@ffff8cb999c68f00 x1874672572561792/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828823 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828798.850857:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c68f00 x1874672572561792/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828823 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828798.850863:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828798.850865:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787828798.850867:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c68f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572561792:192.168.203.129@tcp:38:kworker.0 00010000:00010000:2.0:1787828799.682416:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787828800.706515:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787828801.730390:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787828802.754423:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787828803.778539:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787828803.970478:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828803.970492:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787828803.970498:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828803.970511:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 1756 00000100:00080000:0.0:1787828803.970528:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828803.970537:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:0.0:1787828803.970592:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c69a40 x1874672572561920/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828803.970638:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828803.970644:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787828803.970648:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828803.970664:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 1756 00000100:00080000:0.0:1787828803.970672:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828803.970680:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:0.0:1787828803.970710:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787828803.970737:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c6b840 x1874672572562048/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787828803.970753:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c69a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572561920:192.168.203.129@tcp:250:kworker.0 00000100:00080000:0.0:1787828803.970756:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828803.970774:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1787828803.970778:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828803.970797:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 1756 00000100:00080000:0.0:1787828803.970804:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828803.970812:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:0.0:1787828803.970851:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787828803.970886:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c68f00 x1874672572562176/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1787828803.970888:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787828803.970996:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828803/real 1787828803] req@ffff8cb999c69a40 x1874672572561920/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828828 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828803.971024:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c69a40 x1874672572561920/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828828 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828803.971043:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828803.971049:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787828803.971053:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c69a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572561920:192.168.203.129@tcp:250:kworker.0 00000100:00100000:2.0:1787828803.971082:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c6b840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572562048:192.168.203.129@tcp:38:kworker.0 00000800:00000100:2.0:1787828803.971111:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787828803.971124:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828803/real 1787828803] req@ffff8cb999c6b840 x1874672572562048/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828828 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828803.971162:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c6b840 x1874672572562048/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828828 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828803.971171:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828803.971174:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787828803.971178:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c6b840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572562048:192.168.203.129@tcp:38:kworker.0 00000100:00100000:2.0:1787828803.971184:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c68f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572562176:192.168.203.129@tcp:38:kworker.0 00000800:00000100:2.0:1787828803.971206:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787828803.971217:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828803/real 1787828803] req@ffff8cb999c68f00 x1874672572562176/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828828 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828803.971229:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c68f00 x1874672572562176/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828828 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828803.971237:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828803.971240:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787828803.971243:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c68f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572562176:192.168.203.129@tcp:38:kworker.0 00010000:00010000:1.0:1787828804.802454:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787828805.826478:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787828806.850426:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787828807.874381:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787828808.898516:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787828809.090501:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828809.090517:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787828809.090524:0:37:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828809.090540:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 1761 00000100:00080000:2.0:1787828809.090560:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828809.090616:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:2.0:1787828809.090688:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8e580 x1874672572562304/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828809.090742:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828809.090748:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787828809.090751:0:37:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828809.090756:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 1761 00000100:00080000:2.0:1787828809.090763:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828809.090768:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:2.0:1787828809.090783:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787828809.090800:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8d680 x1874672572562432/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828809.090813:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828809.090819:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1787828809.090822:0:37:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828809.090827:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 1761 00000100:00080000:2.0:1787828809.090831:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828809.090841:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:2.0:1787828809.090848:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787828809.090861:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8cb40 x1874672572562560/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787828809.090904:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8e580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572562304:192.168.203.129@tcp:250:kworker.0 00000800:00000100:0.0:1787828809.091033:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787828809.091102:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828809/real 1787828809] req@ffff8cb998a8e580 x1874672572562304/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828834 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828809.091128:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8e580 x1874672572562304/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828834 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828809.091143:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828809.091148:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787828809.091154:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8e580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572562304:192.168.203.129@tcp:250:kworker.0 00000100:00100000:0.0:1787828809.091168:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8d680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572562432:192.168.203.129@tcp:38:kworker.0 00000800:00000100:0.0:1787828809.091198:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787828809.091214:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828809/real 1787828809] req@ffff8cb998a8d680 x1874672572562432/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828834 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828809.091229:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8d680 x1874672572562432/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828834 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828809.091239:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828809.091243:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787828809.091247:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8d680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572562432:192.168.203.129@tcp:38:kworker.0 00000100:00100000:0.0:1787828809.091316:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8cb40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572562560:192.168.203.129@tcp:38:kworker.0 00000800:00000100:0.0:1787828809.091369:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787828809.091385:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828809/real 1787828809] req@ffff8cb998a8cb40 x1874672572562560/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828834 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828809.091400:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8cb40 x1874672572562560/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828834 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828809.091411:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828809.091415:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787828809.091419:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8cb40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572562560:192.168.203.129@tcp:38:kworker.0 00010000:00010000:2.0:1787828809.922509:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787828810.946460:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787828811.970417:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787828812.994420:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787828814.018487:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787828814.210344:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828814.210352:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787828814.210355:0:37:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828814.210365:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 1766 00000100:00080000:2.0:1787828814.210374:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828814.210380:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:2.0:1787828814.210411:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8c000 x1874672572562688/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828814.210439:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828814.210442:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787828814.210444:0:37:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828814.210448:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 1766 00000100:00080000:2.0:1787828814.210450:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828814.210453:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:2.0:1787828814.210461:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787828814.210472:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8f480 x1874672572562816/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828814.210481:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1787828814.210486:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572562688:192.168.203.129@tcp:250:kworker.0 00000100:00080000:2.0:1787828814.210486:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1787828814.210488:0:37:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828814.210492:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 1766 00000100:00080000:2.0:1787828814.210496:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828814.210502:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:2.0:1787828814.210505:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787828814.210524:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8e940 x1874672572562944/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1787828814.210560:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787828814.210607:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828814/real 1787828814] req@ffff8cb998a8c000 x1874672572562688/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828839 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828814.210624:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8c000 x1874672572562688/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828839 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828814.210632:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828814.210636:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787828814.210647:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572562688:192.168.203.129@tcp:250:kworker.0 00000100:00100000:1.0:1787828814.210668:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572562816:192.168.203.129@tcp:38:kworker.0 00000800:00000100:1.0:1787828814.210688:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787828814.210703:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828814/real 1787828814] req@ffff8cb998a8f480 x1874672572562816/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828839 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828814.210718:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8f480 x1874672572562816/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828839 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828814.210730:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828814.210736:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787828814.210748:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572562816:192.168.203.129@tcp:38:kworker.0 00000100:00100000:1.0:1787828814.210767:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8e940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572562944:192.168.203.129@tcp:38:kworker.0 00000800:00000100:1.0:1787828814.210786:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787828814.210799:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828814/real 1787828814] req@ffff8cb998a8e940 x1874672572562944/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828839 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828814.210822:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8e940 x1874672572562944/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828839 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828814.210840:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828814.210844:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787828814.210848:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8e940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572562944:192.168.203.129@tcp:38:kworker.0 00010000:00010000:1.0:1787828815.042474:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787828816.066497:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787828817.090373:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787828818.114510:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787828819.138431:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787828819.330368:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828819.330378:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787828819.330383:0:37:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828819.330394:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 1771 00000100:00080000:0.0:1787828819.330406:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828819.330415:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:0.0:1787828819.330458:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c6bc00 x1874672572563072/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828819.330486:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828819.330489:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787828819.330490:0:37:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828819.330493:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 1771 00000100:00080000:0.0:1787828819.330495:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828819.330498:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:0.0:1787828819.330508:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787828819.330529:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c69e00 x1874672572563200/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787828819.330539:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c6bc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572563072:192.168.203.129@tcp:250:kworker.0 00000100:00080000:0.0:1787828819.330540:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828819.330549:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1787828819.330550:0:37:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828819.330554:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 1771 00000100:00080000:0.0:1787828819.330556:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828819.330559:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:0.0:1787828819.330562:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787828819.330572:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c6b480 x1874672572563328/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1787828819.330621:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787828819.330665:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828819/real 1787828819] req@ffff8cb999c6bc00 x1874672572563072/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828844 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828819.330681:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c6bc00 x1874672572563072/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828844 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828819.330691:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828819.330694:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787828819.330697:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c6bc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572563072:192.168.203.129@tcp:250:kworker.0 00000100:00100000:1.0:1787828819.330721:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c69e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572563200:192.168.203.129@tcp:38:kworker.0 00000800:00000100:1.0:1787828819.330738:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787828819.330747:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828819/real 1787828819] req@ffff8cb999c69e00 x1874672572563200/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828844 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828819.330781:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c69e00 x1874672572563200/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828844 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828819.330788:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828819.330790:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787828819.330792:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c69e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572563200:192.168.203.129@tcp:38:kworker.0 00000100:00100000:1.0:1787828819.330796:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c6b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572563328:192.168.203.129@tcp:38:kworker.0 00000800:00000100:1.0:1787828819.330812:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787828819.330819:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828819/real 1787828819] req@ffff8cb999c6b480 x1874672572563328/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828844 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828819.330827:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c6b480 x1874672572563328/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828844 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828819.330832:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828819.330834:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787828819.330836:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c6b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572563328:192.168.203.129@tcp:38:kworker.0 00010000:00010000:1.0:1787828820.162428:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787828821.186446:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787828822.210431:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787828823.234384:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787828824.258473:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787828824.450381:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828824.450396:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787828824.450401:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828824.450412:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 1776 00000100:00080000:0.0:1787828824.450424:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828824.450432:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:0.0:1787828824.450481:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c692c0 x1874672572563456/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828824.450511:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828824.450514:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787828824.450515:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828824.450518:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 1776 00000100:00080000:0.0:1787828824.450521:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828824.450544:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:0.0:1787828824.450554:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787828824.450565:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c69680 x1874672572563584/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828824.450575:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1787828824.450576:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c692c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572563456:192.168.203.129@tcp:250:kworker.0 00000100:00080000:0.0:1787828824.450581:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1787828824.450585:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828824.450589:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 1776 00000100:00080000:0.0:1787828824.450591:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828824.450594:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:0.0:1787828824.450597:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787828824.450607:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c68000 x1874672572563712/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1787828824.450654:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787828824.450718:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828824/real 1787828824] req@ffff8cb999c692c0 x1874672572563456/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828849 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828824.450748:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c692c0 x1874672572563456/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828849 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828824.450758:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828824.450761:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787828824.450780:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c692c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572563456:192.168.203.129@tcp:250:kworker.0 00000100:00100000:1.0:1787828824.450810:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c69680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572563584:192.168.203.129@tcp:38:kworker.0 00000800:00000100:1.0:1787828824.450868:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787828824.450880:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828824/real 1787828824] req@ffff8cb999c69680 x1874672572563584/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828849 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828824.450888:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c69680 x1874672572563584/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828849 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828824.450894:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828824.450897:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787828824.450899:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c69680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572563584:192.168.203.129@tcp:38:kworker.0 00000100:00100000:1.0:1787828824.450903:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c68000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572563712:192.168.203.129@tcp:38:kworker.0 00000800:00000100:1.0:1787828824.450934:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787828824.450941:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828824/real 1787828824] req@ffff8cb999c68000 x1874672572563712/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828849 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828824.450948:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c68000 x1874672572563712/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828849 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828824.450962:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828824.450965:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787828824.450974:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c68000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572563712:192.168.203.129@tcp:38:kworker.0 00010000:00010000:2.0:1787828825.282387:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787828826.306513:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787828827.330468:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787828828.354386:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787828829.378529:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787828829.570404:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828829.570420:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787828829.570426:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828829.570440:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 1781 00000100:00080000:2.0:1787828829.570456:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828829.570467:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:2.0:1787828829.570519:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8ed00 x1874672572563840/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828829.570553:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828829.570559:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787828829.570561:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828829.570566:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 1781 00000100:00080000:2.0:1787828829.570572:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828829.570579:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:2.0:1787828829.570593:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787828829.570603:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8ed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572563840:192.168.203.129@tcp:250:kworker.0 00000100:00100000:2.0:1787828829.570611:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8f840 x1874672572563968/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828829.570622:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828829.570644:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1787828829.570646:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828829.570651:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 1781 00000100:00080000:2.0:1787828829.570655:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828829.570659:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:2.0:1787828829.570663:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000800:00000100:1.0:1787828829.570679:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787828829.570682:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8e940 x1874672572564096/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787828829.570728:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828829/real 1787828829] req@ffff8cb998a8ed00 x1874672572563840/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828854 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828829.570746:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8ed00 x1874672572563840/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828854 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828829.570756:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828829.570759:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787828829.570763:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8ed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572563840:192.168.203.129@tcp:250:kworker.0 00000100:00100000:1.0:1787828829.570790:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8f840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572563968:192.168.203.129@tcp:38:kworker.0 00000800:00000100:1.0:1787828829.570826:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787828829.570838:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828829/real 1787828829] req@ffff8cb998a8f840 x1874672572563968/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828854 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828829.570848:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8f840 x1874672572563968/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828854 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828829.570866:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828829.570869:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787828829.570872:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8f840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572563968:192.168.203.129@tcp:38:kworker.0 00000100:00100000:1.0:1787828829.570878:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8e940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572564096:192.168.203.129@tcp:38:kworker.0 00000800:00000100:1.0:1787828829.570895:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787828829.570903:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828829/real 1787828829] req@ffff8cb998a8e940 x1874672572564096/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828854 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828829.570912:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8e940 x1874672572564096/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828854 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828829.570919:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828829.570922:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787828829.570924:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8e940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572564096:192.168.203.129@tcp:38:kworker.0 00010000:00010000:1.0:1787828830.402447:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787828831.426464:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787828832.450406:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787828833.474487:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787828834.498453:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787828834.690345:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828834.690353:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787828834.690356:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828834.690365:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 1786 00000100:00080000:2.0:1787828834.690375:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828834.690380:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:2.0:1787828834.690412:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8f480 x1874672572564224/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828834.690434:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828834.690437:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787828834.690439:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828834.690441:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 1786 00000100:00080000:2.0:1787828834.690444:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828834.690446:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:2.0:1787828834.690453:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787828834.690476:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8c000 x1874672572564352/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828834.690485:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828834.690488:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1787828834.690490:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828834.690497:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 1786 00000100:00100000:1.0:1787828834.690500:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572564224:192.168.203.129@tcp:250:kworker.0 00000100:00080000:2.0:1787828834.690502:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828834.690512:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:2.0:1787828834.690516:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787828834.690528:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8cb40 x1874672572564480/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1787828834.690618:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787828834.690683:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828834/real 1787828834] req@ffff8cb998a8f480 x1874672572564224/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828859 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828834.690710:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8f480 x1874672572564224/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828859 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828834.690747:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828834.690753:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787828834.690791:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572564224:192.168.203.129@tcp:250:kworker.0 00000100:00100000:1.0:1787828834.690824:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572564352:192.168.203.129@tcp:38:kworker.0 00000800:00000100:1.0:1787828834.690851:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787828834.690863:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828834/real 1787828834] req@ffff8cb998a8c000 x1874672572564352/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828859 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828834.690872:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8c000 x1874672572564352/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828859 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828834.690878:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828834.690881:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787828834.690883:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572564352:192.168.203.129@tcp:38:kworker.0 00000100:00100000:1.0:1787828834.690889:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8cb40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572564480:192.168.203.129@tcp:38:kworker.0 00000800:00000100:1.0:1787828834.690927:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787828834.690943:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828834/real 1787828834] req@ffff8cb998a8cb40 x1874672572564480/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828859 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828834.690960:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8cb40 x1874672572564480/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828859 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828834.690971:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828834.690976:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787828834.690979:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8cb40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572564480:192.168.203.129@tcp:38:kworker.0 00010000:00010000:1.0:1787828835.522487:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787828836.546462:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787828837.570482:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787828838.594441:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787828839.618470:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787828839.810352:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828839.810365:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787828839.810370:0:37:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828839.810383:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 1791 00000100:00080000:2.0:1787828839.810396:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828839.810405:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:2.0:1787828839.810453:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8d680 x1874672572564608/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828839.810487:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828839.810491:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787828839.810493:0:37:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828839.810497:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 1791 00000100:00080000:2.0:1787828839.810519:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828839.810525:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:1.0:1787828839.810534:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8d680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572564608:192.168.203.129@tcp:250:kworker.0 00000008:00100000:2.0:1787828839.810538:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787828839.810556:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8e580 x1874672572564736/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828839.810569:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828839.810573:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1787828839.810575:0:37:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828839.810579:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 1791 00000100:00080000:2.0:1787828839.810583:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828839.810587:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:2.0:1787828839.810593:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000800:00000100:1.0:1787828839.810605:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787828839.810608:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8cf00 x1874672572564864/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787828839.810654:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828839/real 1787828839] req@ffff8cb998a8d680 x1874672572564608/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828864 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828839.810668:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8d680 x1874672572564608/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828864 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828839.810677:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828839.810680:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787828839.810683:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8d680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572564608:192.168.203.129@tcp:250:kworker.0 00000100:00100000:1.0:1787828839.810707:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8e580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572564736:192.168.203.129@tcp:38:kworker.0 00000800:00000100:1.0:1787828839.810736:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787828839.810747:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828839/real 1787828839] req@ffff8cb998a8e580 x1874672572564736/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828864 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828839.810756:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8e580 x1874672572564736/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828864 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828839.810762:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828839.810764:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787828839.810767:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8e580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572564736:192.168.203.129@tcp:38:kworker.0 00000100:00100000:1.0:1787828839.810771:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8cf00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572564864:192.168.203.129@tcp:38:kworker.0 00000800:00000100:1.0:1787828839.810795:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787828839.810803:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828839/real 1787828839] req@ffff8cb998a8cf00 x1874672572564864/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828864 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828839.810811:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8cf00 x1874672572564864/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828864 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828839.810817:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828839.810819:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787828839.810821:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8cf00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572564864:192.168.203.129@tcp:38:kworker.0 00010000:00010000:1.0:1787828840.642442:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787828841.666403:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787828842.690466:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787828843.714455:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787828844.738396:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787828844.930376:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828844.930385:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787828844.930388:0:37:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828844.930396:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 1797 00000100:00080000:0.0:1787828844.930411:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828844.930417:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:0.0:1787828844.930447:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c6a580 x1874672572564992/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828844.930467:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828844.930484:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787828844.930486:0:37:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828844.930491:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 1797 00000100:00080000:0.0:1787828844.930496:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828844.930502:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:0.0:1787828844.930512:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787828844.930514:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c6a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572564992:192.168.203.129@tcp:250:kworker.0 00000100:00100000:0.0:1787828844.930529:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c6a1c0 x1874672572565120/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828844.930539:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828844.930542:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1787828844.930544:0:37:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828844.930547:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 1797 00000100:00080000:0.0:1787828844.930551:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828844.930553:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:0.0:1787828844.930557:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787828844.930567:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c68000 x1874672572565248/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787828844.930616:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c6a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572565120:192.168.203.129@tcp:38:kworker.0 00000100:00100000:1.0:1787828844.930636:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c68000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572565248:192.168.203.129@tcp:38:kworker.0 00010000:00010000:2.0:1787828845.762582:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787828846.786421:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787828847.810543:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:3.0:1787828848.002685:0:2383:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.129:988 00000400:00000100:3.0:1787828848.002736:0:2383:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.129@tcp at host 192.168.203.129 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:3.0:1787828848.002755:0:2383:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.129@tcp 00000800:00000100:3.0:1787828848.002799:0:2383:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.129@tcp 00000800:00000100:3.0:1787828848.002812:0:2383:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.129@tcp 00000100:00100000:1.0:1787828848.002869:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828844/real 1787828848] req@ffff8cb999c6a1c0 x1874672572565120/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828869 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828848.002900:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c6a1c0 x1874672572565120/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828869 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828848.002915:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828848.002921:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787828848.002949:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c6a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572565120:192.168.203.129@tcp:38:kworker.0 00000100:00100000:1.0:1787828848.002959:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828844/real 1787828848] req@ffff8cb999c68000 x1874672572565248/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828869 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828848.002988:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c68000 x1874672572565248/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828869 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828848.002995:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828848.002999:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787828848.003003:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c68000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572565248:192.168.203.129@tcp:38:kworker.0 00000100:00100000:1.0:1787828848.003010:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828844/real 1787828848] req@ffff8cb999c6a580 x1874672572564992/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828869 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828848.003021:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c6a580 x1874672572564992/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828869 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828848.003028:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828848.003032:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787828848.003036:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c6a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572564992:192.168.203.129@tcp:250:kworker.0 00010000:00010000:0.0:1787828848.834476:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787828849.858696:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787828850.050388:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828850.050399:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787828850.050403:0:37:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828850.050414:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 1802 00000100:00080000:0.0:1787828850.050427:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828850.050434:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:0.0:1787828850.050493:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c69680 x1874672572565376/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828850.050531:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828850.050534:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787828850.050536:0:37:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828850.050539:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 1802 00000100:00080000:0.0:1787828850.050542:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828850.050546:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:0.0:1787828850.050555:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787828850.050564:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c692c0 x1874672572565504/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828850.050572:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828850.050575:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1787828850.050577:0:37:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828850.050582:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 1802 00000100:00080000:0.0:1787828850.050586:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828850.050590:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:1.0:1787828850.050597:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c69680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572565376:192.168.203.129@tcp:250:kworker.0 00000008:00100000:0.0:1787828850.050598:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787828850.050628:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c6b480 x1874672572565632/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1787828850.050720:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787828850.050788:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828850/real 1787828850] req@ffff8cb999c69680 x1874672572565376/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828875 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828850.050815:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c69680 x1874672572565376/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828875 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828850.050846:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828850.050852:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787828850.050858:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c69680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572565376:192.168.203.129@tcp:250:kworker.0 00000100:00100000:1.0:1787828850.050867:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c692c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572565504:192.168.203.129@tcp:38:kworker.0 00000800:00000100:1.0:1787828850.050901:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787828850.050919:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828850/real 1787828850] req@ffff8cb999c692c0 x1874672572565504/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828875 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828850.050932:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c692c0 x1874672572565504/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828875 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828850.050940:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828850.050944:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787828850.050947:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c692c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572565504:192.168.203.129@tcp:38:kworker.0 00000100:00100000:1.0:1787828850.050980:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c6b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572565632:192.168.203.129@tcp:38:kworker.0 00000800:00000100:1.0:1787828850.051006:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787828850.051019:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828850/real 1787828850] req@ffff8cb999c6b480 x1874672572565632/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828875 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828850.051032:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c6b480 x1874672572565632/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828875 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828850.051040:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828850.051044:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787828850.051047:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c6b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572565632:192.168.203.129@tcp:38:kworker.0 00010000:00010000:3.0:1787828850.882809:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787828851.906691:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787828852.930770:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787828853.954569:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787828854.978512:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787828855.170382:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828855.170390:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787828855.170405:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828855.170413:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 1807 00000100:00080000:2.0:1787828855.170421:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828855.170427:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:2.0:1787828855.170465:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8c780 x1874672572565760/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828855.170529:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828855.170534:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787828855.170536:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828855.170541:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 1807 00000100:00080000:2.0:1787828855.170546:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828855.170551:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:2.0:1787828855.170562:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787828855.170577:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8f0c0 x1874672572565888/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787828855.170582:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8c780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572565760:192.168.203.129@tcp:250:kworker.0 00000100:00080000:2.0:1787828855.170590:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828855.170594:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1787828855.170596:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828855.170600:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 1807 00000100:00080000:2.0:1787828855.170603:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828855.170607:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:2.0:1787828855.170612:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787828855.170639:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8fc00 x1874672572566016/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1787828855.170685:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787828855.170725:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828855/real 1787828855] req@ffff8cb998a8c780 x1874672572565760/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828880 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828855.170738:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8c780 x1874672572565760/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828880 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828855.170746:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828855.170749:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787828855.170752:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8c780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572565760:192.168.203.129@tcp:250:kworker.0 00000100:00100000:1.0:1787828855.170771:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8f0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572565888:192.168.203.129@tcp:38:kworker.0 00000800:00000100:1.0:1787828855.170789:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787828855.170798:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828855/real 1787828855] req@ffff8cb998a8f0c0 x1874672572565888/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828880 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828855.170806:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8f0c0 x1874672572565888/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828880 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828855.170812:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828855.170815:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787828855.170817:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8f0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572565888:192.168.203.129@tcp:38:kworker.0 00000100:00100000:1.0:1787828855.170821:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8fc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572566016:192.168.203.129@tcp:38:kworker.0 00000800:00000100:1.0:1787828855.170843:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787828855.170850:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828855/real 1787828855] req@ffff8cb998a8fc00 x1874672572566016/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828880 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828855.170859:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8fc00 x1874672572566016/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828880 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828855.170865:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828855.170867:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787828855.170869:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8fc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572566016:192.168.203.129@tcp:38:kworker.0 00010000:00010000:2.0:1787828856.002772:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787828857.026681:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787828858.050440:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787828859.074484:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787828860.098430:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787828860.290481:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828860.290495:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787828860.290501:0:37:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828860.290544:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 1812 00000100:00080000:0.0:1787828860.290560:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828860.290569:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:0.0:1787828860.290625:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c69e00 x1874672572566144/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828860.290657:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828860.290662:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787828860.290664:0:37:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828860.290668:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 1812 00000100:00080000:0.0:1787828860.290672:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828860.290676:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:0.0:1787828860.290689:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787828860.290704:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c6bc00 x1874672572566272/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828860.290717:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1787828860.290720:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c69e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572566144:192.168.203.129@tcp:250:kworker.0 00000100:00080000:0.0:1787828860.290722:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1787828860.290726:0:37:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828860.290733:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 1812 00000100:00080000:0.0:1787828860.290737:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828860.290741:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:0.0:1787828860.290747:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787828860.290760:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c68f00 x1874672572566400/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1787828860.290809:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787828860.290869:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828860/real 1787828860] req@ffff8cb999c69e00 x1874672572566144/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828885 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828860.290891:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c69e00 x1874672572566144/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828885 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828860.290902:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828860.290907:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787828860.290911:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c69e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572566144:192.168.203.129@tcp:250:kworker.0 00000100:00100000:1.0:1787828860.290940:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c6bc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572566272:192.168.203.129@tcp:38:kworker.0 00000800:00000100:1.0:1787828860.290967:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787828860.290982:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828860/real 1787828860] req@ffff8cb999c6bc00 x1874672572566272/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828885 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828860.291012:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c6bc00 x1874672572566272/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828885 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828860.291022:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828860.291026:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787828860.291030:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c6bc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572566272:192.168.203.129@tcp:38:kworker.0 00000100:00100000:1.0:1787828860.291036:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c68f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572566400:192.168.203.129@tcp:38:kworker.0 00000800:00000100:1.0:1787828860.291062:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787828860.291076:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828860/real 1787828860] req@ffff8cb999c68f00 x1874672572566400/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828885 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828860.291090:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c68f00 x1874672572566400/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828885 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828860.291098:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828860.291102:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787828860.291106:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c68f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572566400:192.168.203.129@tcp:38:kworker.0 00010000:00010000:1.0:1787828861.122403:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787828862.146428:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787828863.170392:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787828864.194365:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787828865.218442:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787828865.410378:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828865.410420:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787828865.410427:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828865.410441:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 1817 00000100:00080000:1.0:1787828865.410458:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828865.410470:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:1.0:1787828865.410541:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f38f00 x1874672572566528/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828865.410601:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828865.410606:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787828865.410609:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828865.410615:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 1817 00000100:00080000:1.0:1787828865.410621:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828865.410628:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:1.0:1787828865.410642:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787828865.410647:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f38f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572566528:192.168.203.129@tcp:250:kworker.0 00000100:00100000:1.0:1787828865.410659:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f392c0 x1874672572566656/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828865.410671:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828865.410675:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1787828865.410678:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828865.410682:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 1817 00000100:00080000:1.0:1787828865.410686:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828865.410691:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:1.0:1787828865.410696:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787828865.410708:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f3a580 x1874672572566784/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1787828865.410764:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787828865.410830:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828865/real 1787828865] req@ffff8cb999f38f00 x1874672572566528/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828890 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828865.410844:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f38f00 x1874672572566528/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828890 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828865.410881:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828865.410885:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787828865.410898:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f38f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572566528:192.168.203.129@tcp:250:kworker.0 00000100:00100000:0.0:1787828865.410957:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f392c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572566656:192.168.203.129@tcp:38:kworker.0 00000800:00000100:0.0:1787828865.410977:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787828865.410986:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828865/real 1787828865] req@ffff8cb999f392c0 x1874672572566656/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828890 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828865.411012:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f392c0 x1874672572566656/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828890 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828865.411019:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828865.411023:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787828865.411025:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f392c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572566656:192.168.203.129@tcp:38:kworker.0 00000100:00100000:0.0:1787828865.411030:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f3a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572566784:192.168.203.129@tcp:38:kworker.0 00000800:00000100:0.0:1787828865.411045:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787828865.411064:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828865/real 1787828865] req@ffff8cb999f3a580 x1874672572566784/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828890 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828865.411074:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f3a580 x1874672572566784/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828890 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828865.411081:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828865.411084:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787828865.411086:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f3a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572566784:192.168.203.129@tcp:38:kworker.0 00010000:00010000:3.0:1787828866.242524:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787828867.266430:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787828868.290487:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787828869.314361:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787828870.338465:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787828870.530430:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828870.530456:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787828870.530460:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828870.530469:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 1822 00000100:00080000:1.0:1787828870.530478:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828870.530484:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:1.0:1787828870.530530:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f383c0 x1874672572566912/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828870.530551:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828870.530554:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787828870.530555:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828870.530558:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 1822 00000100:00080000:1.0:1787828870.530561:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828870.530564:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:1.0:1787828870.530573:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787828870.530580:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f383c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572566912:192.168.203.129@tcp:250:kworker.0 00000100:00100000:1.0:1787828870.530586:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f3a580 x1874672572567040/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828870.530594:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828870.530596:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1787828870.530598:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828870.530600:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 1822 00000100:00080000:1.0:1787828870.530602:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828870.530604:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:1.0:1787828870.530607:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787828870.530615:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f392c0 x1874672572567168/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1787828870.530637:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787828870.530676:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828870/real 1787828870] req@ffff8cb999f383c0 x1874672572566912/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828895 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828870.530691:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f383c0 x1874672572566912/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828895 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828870.530700:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828870.530703:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787828870.530706:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f383c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572566912:192.168.203.129@tcp:250:kworker.0 00000100:00100000:0.0:1787828870.530724:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f3a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572567040:192.168.203.129@tcp:38:kworker.0 00000800:00000100:0.0:1787828870.530738:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787828870.530746:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828870/real 1787828870] req@ffff8cb999f3a580 x1874672572567040/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828895 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828870.530754:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f3a580 x1874672572567040/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828895 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828870.530774:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828870.530777:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787828870.530781:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f3a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572567040:192.168.203.129@tcp:38:kworker.0 00000100:00100000:0.0:1787828870.530787:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f392c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572567168:192.168.203.129@tcp:38:kworker.0 00000800:00000100:0.0:1787828870.530809:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787828870.530821:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828870/real 1787828870] req@ffff8cb999f392c0 x1874672572567168/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828895 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828870.530831:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f392c0 x1874672572567168/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828895 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828870.530837:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828870.530839:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787828870.530841:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f392c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572567168:192.168.203.129@tcp:38:kworker.0 00010000:00010000:2.0:1787828871.362388:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787828872.386392:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787828873.410401:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787828874.434403:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787828875.458403:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787828875.650322:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828875.650330:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787828875.650333:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828875.650341:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 1827 00000100:00080000:1.0:1787828875.650351:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828875.650356:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:1.0:1787828875.650387:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f38f00 x1874672572567296/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828875.650409:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828875.650411:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787828875.650413:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828875.650415:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 1827 00000100:00080000:1.0:1787828875.650417:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828875.650420:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:1.0:1787828875.650427:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787828875.650434:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f3b0c0 x1874672572567424/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828875.650441:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828875.650443:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1787828875.650445:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828875.650447:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 1827 00000100:00080000:1.0:1787828875.650448:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828875.650465:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:1.0:1787828875.650469:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787828875.650494:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f38f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572567296:192.168.203.129@tcp:250:kworker.0 00000100:00100000:1.0:1787828875.650495:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f3b480 x1874672572567552/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1787828875.650559:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787828875.650593:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828875/real 1787828875] req@ffff8cb999f38f00 x1874672572567296/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828900 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828875.650606:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f38f00 x1874672572567296/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828900 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828875.650613:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828875.650616:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787828875.650632:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f38f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572567296:192.168.203.129@tcp:250:kworker.0 00000100:00100000:0.0:1787828875.650638:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f3b0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572567424:192.168.203.129@tcp:38:kworker.0 00000800:00000100:0.0:1787828875.650655:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787828875.650663:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828875/real 1787828875] req@ffff8cb999f3b0c0 x1874672572567424/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828900 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828875.650671:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f3b0c0 x1874672572567424/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828900 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828875.650677:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828875.650679:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787828875.650681:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f3b0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572567424:192.168.203.129@tcp:38:kworker.0 00000100:00100000:0.0:1787828875.650702:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f3b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572567552:192.168.203.129@tcp:38:kworker.0 00000800:00000100:0.0:1787828875.650712:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787828875.650719:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828875/real 1787828875] req@ffff8cb999f3b480 x1874672572567552/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828900 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828875.650726:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f3b480 x1874672572567552/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828900 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828875.650731:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828875.650733:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787828875.650735:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f3b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572567552:192.168.203.129@tcp:38:kworker.0 00010000:00010000:2.0:1787828876.482446:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787828877.506449:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787828878.530395:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787828879.554408:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787828880.578408:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787828880.770374:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828880.770381:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787828880.770400:0:37:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828880.770408:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 1832 00000100:00080000:0.0:1787828880.770417:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828880.770423:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:0.0:1787828880.770468:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c6b840 x1874672572567680/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828880.770492:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828880.770495:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787828880.770497:0:37:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828880.770499:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 1832 00000100:00080000:0.0:1787828880.770502:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828880.770504:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:0.0:1787828880.770512:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787828880.770521:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c69a40 x1874672572567808/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828880.770529:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828880.770532:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1787828880.770534:0:37:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1787828880.770539:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c6b840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572567680:192.168.203.129@tcp:250:kworker.0 00000100:00080000:0.0:1787828880.770540:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 1832 00000100:00080000:0.0:1787828880.770545:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828880.770551:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:0.0:1787828880.770554:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787828880.770563:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c68f00 x1874672572567936/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1787828880.770635:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787828880.770688:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828880/real 1787828880] req@ffff8cb999c6b840 x1874672572567680/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828905 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828880.770710:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c6b840 x1874672572567680/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828905 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828880.770721:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828880.770726:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787828880.770731:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c6b840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572567680:192.168.203.129@tcp:250:kworker.0 00000100:00100000:1.0:1787828880.770756:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c69a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572567808:192.168.203.129@tcp:38:kworker.0 00000800:00000100:1.0:1787828880.770781:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787828880.770796:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828880/real 1787828880] req@ffff8cb999c69a40 x1874672572567808/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828905 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828880.770810:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c69a40 x1874672572567808/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828905 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828880.770819:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828880.770823:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787828880.770828:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c69a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572567808:192.168.203.129@tcp:38:kworker.0 00000100:00100000:1.0:1787828880.770852:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c68f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572567936:192.168.203.129@tcp:38:kworker.0 00000800:00000100:1.0:1787828880.770878:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787828880.770892:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828880/real 1787828880] req@ffff8cb999c68f00 x1874672572567936/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828905 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828880.770906:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c68f00 x1874672572567936/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828905 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828880.770916:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828880.770920:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787828880.770924:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c68f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572567936:192.168.203.129@tcp:38:kworker.0 00010000:00010000:3.0:1787828881.602350:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787828882.626435:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787828883.650380:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787828884.674368:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787828885.698393:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787828885.890340:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828885.890348:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787828885.890353:0:37:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828885.890361:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 1838 00000100:00080000:0.0:1787828885.890370:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828885.890377:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:0.0:1787828885.890419:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c6bc00 x1874672572568064/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828885.890456:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828885.890459:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787828885.890461:0:37:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828885.890464:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 1838 00000100:00080000:0.0:1787828885.890467:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828885.890470:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:0.0:1787828885.890478:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787828885.890491:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c69e00 x1874672572568192/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828885.890500:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828885.890503:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1787828885.890508:0:37:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1787828885.890508:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c6bc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572568064:192.168.203.129@tcp:250:kworker.0 00000100:00080000:0.0:1787828885.890516:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 1838 00000100:00080000:0.0:1787828885.890519:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828885.890522:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:0.0:1787828885.890534:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787828885.890546:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c6b480 x1874672572568320/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1787828885.890571:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787828885.890608:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828885/real 1787828885] req@ffff8cb999c6bc00 x1874672572568064/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828910 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828885.890622:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c6bc00 x1874672572568064/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828910 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828885.890630:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828885.890633:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787828885.890636:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c6bc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572568064:192.168.203.129@tcp:250:kworker.0 00000100:00100000:2.0:1787828885.890656:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c69e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572568192:192.168.203.129@tcp:38:kworker.0 00000800:00000100:2.0:1787828885.890671:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787828885.890681:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828885/real 1787828885] req@ffff8cb999c69e00 x1874672572568192/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828910 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828885.890692:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c69e00 x1874672572568192/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828910 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828885.890698:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828885.890707:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787828885.890710:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c69e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572568192:192.168.203.129@tcp:38:kworker.0 00000100:00100000:2.0:1787828885.890715:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c6b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572568320:192.168.203.129@tcp:38:kworker.0 00000800:00000100:2.0:1787828885.890729:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787828885.890736:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828885/real 1787828885] req@ffff8cb999c6b480 x1874672572568320/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828910 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828885.890745:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c6b480 x1874672572568320/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828910 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828885.890750:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828885.890752:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787828885.890754:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c6b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572568320:192.168.203.129@tcp:38:kworker.0 00010000:00010000:1.0:1787828886.722414:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787828887.746361:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787828888.770470:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787828889.794462:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787828890.818459:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787828891.010344:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828891.010352:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787828891.010356:0:37:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828891.010377:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 1843 00000100:00080000:0.0:1787828891.010413:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828891.010421:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:0.0:1787828891.010467:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c692c0 x1874672572568448/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828891.010490:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828891.010493:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787828891.010495:0:37:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828891.010498:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 1843 00000100:00080000:0.0:1787828891.010500:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828891.010503:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:0.0:1787828891.010511:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787828891.010534:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c69680 x1874672572568576/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828891.010543:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828891.010546:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1787828891.010547:0:37:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828891.010552:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 1843 00000100:00100000:2.0:1787828891.010557:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c692c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572568448:192.168.203.129@tcp:250:kworker.0 00000100:00080000:0.0:1787828891.010569:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828891.010574:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:0.0:1787828891.010579:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787828891.010589:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c6a580 x1874672572568704/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1787828891.010625:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787828891.010702:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828891/real 1787828891] req@ffff8cb999c692c0 x1874672572568448/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828916 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828891.010730:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c692c0 x1874672572568448/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828916 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828891.010746:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828891.010752:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787828891.010759:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c692c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572568448:192.168.203.129@tcp:250:kworker.0 00000100:00100000:2.0:1787828891.010814:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c69680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572568576:192.168.203.129@tcp:38:kworker.0 00000800:00000100:2.0:1787828891.010862:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787828891.010883:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828891/real 1787828891] req@ffff8cb999c69680 x1874672572568576/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828916 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828891.010901:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c69680 x1874672572568576/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828916 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828891.010909:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828891.010913:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787828891.010916:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c69680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572568576:192.168.203.129@tcp:38:kworker.0 00000100:00100000:2.0:1787828891.010939:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c6a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572568704:192.168.203.129@tcp:38:kworker.0 00000800:00000100:2.0:1787828891.010954:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787828891.010961:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828891/real 1787828891] req@ffff8cb999c6a580 x1874672572568704/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828916 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828891.010969:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c6a580 x1874672572568704/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828916 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828891.010974:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828891.010976:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787828891.010978:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c6a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572568704:192.168.203.129@tcp:38:kworker.0 00010000:00010000:2.0:1787828891.842443:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787828892.866468:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787828893.890407:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787828894.914402:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787828895.938481:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787828896.130411:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828896.130423:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787828896.130428:0:37:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828896.130454:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 1848 00000100:00080000:2.0:1787828896.130465:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828896.130472:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:2.0:1787828896.130517:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8e1c0 x1874672572568832/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828896.130546:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828896.130550:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787828896.130553:0:37:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828896.130557:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 1848 00000100:00080000:2.0:1787828896.130562:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828896.130568:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:2.0:1787828896.130590:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787828896.130598:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8e1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572568832:192.168.203.129@tcp:250:kworker.0 00000100:00100000:2.0:1787828896.130607:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8da40 x1874672572568960/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828896.130620:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828896.130625:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1787828896.130627:0:37:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828896.130632:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 1848 00000100:00080000:2.0:1787828896.130637:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828896.130641:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:2.0:1787828896.130646:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000800:00000100:1.0:1787828896.130652:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787828896.130662:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8de00 x1874672572569088/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787828896.130693:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828896/real 1787828896] req@ffff8cb998a8e1c0 x1874672572568832/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828921 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828896.130713:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8e1c0 x1874672572568832/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828921 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828896.130734:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828896.130739:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787828896.130743:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8e1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572568832:192.168.203.129@tcp:250:kworker.0 00000100:00100000:1.0:1787828896.130777:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8da40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572568960:192.168.203.129@tcp:38:kworker.0 00000800:00000100:1.0:1787828896.130807:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787828896.130819:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828896/real 1787828896] req@ffff8cb998a8da40 x1874672572568960/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828921 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828896.130832:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8da40 x1874672572568960/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828921 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828896.130841:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828896.130844:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787828896.130859:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8da40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572568960:192.168.203.129@tcp:38:kworker.0 00000100:00100000:1.0:1787828896.130866:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8de00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572569088:192.168.203.129@tcp:38:kworker.0 00000800:00000100:1.0:1787828896.130888:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787828896.130908:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828896/real 1787828896] req@ffff8cb998a8de00 x1874672572569088/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828921 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828896.130921:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8de00 x1874672572569088/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828921 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828896.130927:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828896.130929:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787828896.130931:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8de00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572569088:192.168.203.129@tcp:38:kworker.0 00010000:00010000:3.0:1787828896.962388:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787828897.986480:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787828899.010402:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787828900.034384:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787828901.058405:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787828901.250386:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828901.250395:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787828901.250399:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828901.250407:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 1853 00000100:00080000:3.0:1787828901.250416:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828901.250422:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:3.0:1787828901.250459:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb992317480 x1874672572569216/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828901.250480:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828901.250483:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787828901.250485:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828901.250488:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 1853 00000100:00080000:3.0:1787828901.250490:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828901.250493:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:3.0:1787828901.250501:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787828901.250509:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb992314b40 x1874672572569344/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828901.250534:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828901.250537:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1787828901.250538:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828901.250541:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 1853 00000100:00080000:3.0:1787828901.250543:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828901.250559:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:3.0:1787828901.250564:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787828901.250574:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb992314780 x1874672572569472/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787828901.250591:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb992317480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572569216:192.168.203.129@tcp:250:kworker.0 00000800:00000100:1.0:1787828901.250647:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787828901.250689:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828901/real 1787828901] req@ffff8cb992317480 x1874672572569216/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828926 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828901.250704:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb992317480 x1874672572569216/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828926 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828901.250713:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828901.250717:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787828901.250720:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb992317480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572569216:192.168.203.129@tcp:250:kworker.0 00000100:00100000:1.0:1787828901.250726:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb992314b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572569344:192.168.203.129@tcp:38:kworker.0 00000800:00000100:1.0:1787828901.250744:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787828901.250754:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828901/real 1787828901] req@ffff8cb992314b40 x1874672572569344/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828926 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828901.250764:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb992314b40 x1874672572569344/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828926 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828901.250771:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828901.250773:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787828901.250776:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb992314b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572569344:192.168.203.129@tcp:38:kworker.0 00000100:00100000:1.0:1787828901.250799:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb992314780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572569472:192.168.203.129@tcp:38:kworker.0 00000800:00000100:1.0:1787828901.250813:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787828901.250821:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828901/real 1787828901] req@ffff8cb992314780 x1874672572569472/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828926 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828901.250830:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb992314780 x1874672572569472/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828926 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828901.250848:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828901.250852:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787828901.250855:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb992314780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572569472:192.168.203.129@tcp:38:kworker.0 00010000:00010000:1.0:1787828902.082397:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787828903.106425:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787828904.130520:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787828905.154485:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787828906.178378:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787828906.370399:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828906.370410:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787828906.370414:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828906.370424:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 1858 00000100:00080000:3.0:1787828906.370453:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828906.370460:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:3.0:1787828906.370522:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb992316580 x1874672572569600/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828906.370547:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828906.370550:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787828906.370552:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828906.370555:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 1858 00000100:00080000:3.0:1787828906.370558:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828906.370561:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:3.0:1787828906.370569:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787828906.370581:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb992314780 x1874672572569728/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828906.370590:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828906.370613:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00100000:1.0:1787828906.370614:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb992316580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572569600:192.168.203.129@tcp:250:kworker.0 00000100:00080000:3.0:1787828906.370617:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828906.370630:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 1858 00000100:00080000:3.0:1787828906.370637:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828906.370643:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:3.0:1787828906.370651:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787828906.370688:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb992314b40 x1874672572569856/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1787828906.370712:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787828906.370761:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828906/real 1787828906] req@ffff8cb992316580 x1874672572569600/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828931 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828906.370781:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb992316580 x1874672572569600/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828931 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828906.370793:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828906.370797:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787828906.370801:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb992316580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572569600:192.168.203.129@tcp:250:kworker.0 00000100:00100000:1.0:1787828906.370809:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb992314780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572569728:192.168.203.129@tcp:38:kworker.0 00000800:00000100:1.0:1787828906.370830:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787828906.370843:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828906/real 1787828906] req@ffff8cb992314780 x1874672572569728/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828931 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828906.370870:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb992314780 x1874672572569728/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828931 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828906.370879:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828906.370905:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787828906.370909:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb992314780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572569728:192.168.203.129@tcp:38:kworker.0 00000100:00100000:1.0:1787828906.370957:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb992314b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572569856:192.168.203.129@tcp:38:kworker.0 00000800:00000100:1.0:1787828906.371010:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787828906.371025:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828906/real 1787828906] req@ffff8cb992314b40 x1874672572569856/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828931 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828906.371041:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb992314b40 x1874672572569856/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828931 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828906.371052:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828906.371056:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787828906.371061:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb992314b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572569856:192.168.203.129@tcp:38:kworker.0 00010000:00010000:2.0:1787828907.202482:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787828908.226425:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787828909.250388:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787828910.274399:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787828911.298417:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787828911.490386:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828911.490395:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787828911.490398:0:37:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828911.490406:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 1863 00000100:00080000:0.0:1787828911.490416:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828911.490422:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:0.0:1787828911.490452:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c68000 x1874672572569984/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828911.490475:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828911.490478:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787828911.490479:0:37:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828911.490482:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 1863 00000100:00080000:0.0:1787828911.490484:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828911.490487:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:0.0:1787828911.490494:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787828911.490502:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c6a1c0 x1874672572570112/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828911.490510:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1787828911.490518:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c68000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572569984:192.168.203.129@tcp:250:kworker.0 00000100:00080000:0.0:1787828911.490527:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1787828911.490529:0:37:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828911.490533:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 1863 00000100:00080000:0.0:1787828911.490536:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828911.490539:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:0.0:1787828911.490544:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787828911.490554:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c6a580 x1874672572570240/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787828911.490608:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c6a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572570112:192.168.203.129@tcp:38:kworker.0 00000100:00100000:1.0:1787828911.490628:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c6a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572570240:192.168.203.129@tcp:38:kworker.0 00010000:00010000:3.0:1787828912.322378:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787828913.346425:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787828914.371082:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:0.0:1787828914.562677:0:2384:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.129:988 00000400:00000100:0.0:1787828914.562739:0:2384:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.129@tcp at host 192.168.203.129 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:0.0:1787828914.562761:0:2384:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.129@tcp 00000800:00000100:0.0:1787828914.562815:0:2384:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.129@tcp 00000800:00000100:0.0:1787828914.562831:0:2384:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.129@tcp 00000100:00100000:1.0:1787828914.562847:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828911/real 1787828914] req@ffff8cb999c6a1c0 x1874672572570112/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828936 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828914.562881:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c6a1c0 x1874672572570112/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828936 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828914.562918:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828914.562925:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787828914.562930:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c6a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572570112:192.168.203.129@tcp:38:kworker.0 00000100:00100000:1.0:1787828914.562939:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828911/real 1787828914] req@ffff8cb999c6a580 x1874672572570240/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828936 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828914.562954:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c6a580 x1874672572570240/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828936 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828914.562962:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828914.562967:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787828914.562971:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c6a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572570240:192.168.203.129@tcp:38:kworker.0 00000100:00100000:1.0:1787828914.562978:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828911/real 1787828914] req@ffff8cb999c68000 x1874672572569984/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828936 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828914.562992:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c68000 x1874672572569984/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828936 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828914.563001:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828914.563005:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787828914.563008:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c68000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572569984:192.168.203.129@tcp:250:kworker.0 00010000:00010000:3.0:1787828915.394480:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787828916.418643:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787828916.610391:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828916.610399:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787828916.610402:0:37:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828916.610410:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 1868 00000100:00080000:2.0:1787828916.610418:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828916.610423:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:2.0:1787828916.610460:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8d2c0 x1874672572570368/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828916.610490:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828916.610494:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787828916.610503:0:37:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828916.610507:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 1868 00000100:00080000:2.0:1787828916.610511:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828916.610516:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:2.0:1787828916.610525:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787828916.610541:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8c3c0 x1874672572570496/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787828916.610546:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8d2c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572570368:192.168.203.129@tcp:250:kworker.0 00000100:00080000:2.0:1787828916.610560:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828916.610578:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1787828916.610580:0:37:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828916.610586:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 1868 00000100:00080000:2.0:1787828916.610590:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828916.610595:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:2.0:1787828916.610602:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000800:00000100:1.0:1787828916.610619:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787828916.610624:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8de00 x1874672572570624/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787828916.610664:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828916/real 1787828916] req@ffff8cb998a8d2c0 x1874672572570368/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828941 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828916.610694:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8d2c0 x1874672572570368/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828941 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828916.610705:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828916.610709:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787828916.610713:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8d2c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572570368:192.168.203.129@tcp:250:kworker.0 00000100:00100000:1.0:1787828916.610737:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8c3c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572570496:192.168.203.129@tcp:38:kworker.0 00000800:00000100:1.0:1787828916.610761:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787828916.610775:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828916/real 1787828916] req@ffff8cb998a8c3c0 x1874672572570496/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828941 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828916.610790:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8c3c0 x1874672572570496/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828941 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828916.610799:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828916.610802:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787828916.610805:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8c3c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572570496:192.168.203.129@tcp:38:kworker.0 00000100:00100000:1.0:1787828916.610812:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8de00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572570624:192.168.203.129@tcp:38:kworker.0 00000800:00000100:1.0:1787828916.610834:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787828916.610846:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828916/real 1787828916] req@ffff8cb998a8de00 x1874672572570624/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828941 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828916.610859:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8de00 x1874672572570624/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828941 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828916.610881:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828916.610885:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787828916.610889:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8de00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572570624:192.168.203.129@tcp:38:kworker.0 00010000:00010000:0.0:1787828917.442715:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787828918.466773:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787828919.490803:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787828920.514517:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787828921.538355:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787828921.730364:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828921.730375:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787828921.730379:0:37:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828921.730390:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 1873 00000100:00080000:0.0:1787828921.730403:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828921.730415:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:0.0:1787828921.730455:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c69680 x1874672572570752/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828921.730493:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828921.730498:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787828921.730517:0:37:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828921.730524:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 1873 00000100:00080000:0.0:1787828921.730531:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828921.730536:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:2.0:1787828921.730547:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c69680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572570752:192.168.203.129@tcp:250:kworker.0 00000008:00100000:0.0:1787828921.730549:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787828921.730571:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c692c0 x1874672572570880/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828921.730584:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828921.730587:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1787828921.730589:0:37:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828921.730593:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 1873 00000100:00080000:0.0:1787828921.730596:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828921.730600:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:0.0:1787828921.730604:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787828921.730615:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c6b480 x1874672572571008/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1787828921.730623:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787828921.730665:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828921/real 1787828921] req@ffff8cb999c69680 x1874672572570752/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828946 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828921.730679:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c69680 x1874672572570752/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828946 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828921.730688:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828921.730692:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787828921.730696:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c69680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572570752:192.168.203.129@tcp:250:kworker.0 00000100:00100000:2.0:1787828921.730717:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c692c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572570880:192.168.203.129@tcp:38:kworker.0 00000800:00000100:2.0:1787828921.730733:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787828921.730742:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828921/real 1787828921] req@ffff8cb999c692c0 x1874672572570880/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828946 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828921.730763:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c692c0 x1874672572570880/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828946 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828921.730769:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828921.730772:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787828921.730774:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c692c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572570880:192.168.203.129@tcp:38:kworker.0 00000100:00100000:2.0:1787828921.730779:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c6b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572571008:192.168.203.129@tcp:38:kworker.0 00000800:00000100:2.0:1787828921.730793:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787828921.730811:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828921/real 1787828921] req@ffff8cb999c6b480 x1874672572571008/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828946 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828921.730820:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c6b480 x1874672572571008/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828946 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828921.730827:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828921.730830:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787828921.730832:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c6b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572571008:192.168.203.129@tcp:38:kworker.0 00010000:00010000:2.0:1787828922.563136:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787828923.586633:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787828924.610362:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787828925.634399:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787828926.658422:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787828926.850336:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828926.850345:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787828926.850349:0:37:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828926.850357:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 1879 00000100:00080000:0.0:1787828926.850366:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828926.850372:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:0.0:1787828926.850423:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c69e00 x1874672572571136/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828926.850481:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828926.850488:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787828926.850492:0:37:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828926.850500:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 1879 00000100:00080000:0.0:1787828926.850509:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828926.850518:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:0.0:1787828926.850549:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787828926.850557:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c69e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572571136:192.168.203.129@tcp:250:kworker.0 00000100:00100000:0.0:1787828926.850583:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c6bc00 x1874672572571264/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828926.850614:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828926.850619:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1787828926.850621:0:37:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828926.850626:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 1879 00000100:00080000:0.0:1787828926.850630:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828926.850635:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:0.0:1787828926.850641:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000800:00000100:2.0:1787828926.850649:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787828926.850654:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c68f00 x1874672572571392/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787828926.850739:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828926/real 1787828926] req@ffff8cb999c69e00 x1874672572571136/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828951 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828926.850805:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c69e00 x1874672572571136/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828951 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828926.850814:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828926.850817:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787828926.850821:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c69e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572571136:192.168.203.129@tcp:250:kworker.0 00000100:00100000:2.0:1787828926.850843:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c6bc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572571264:192.168.203.129@tcp:38:kworker.0 00000800:00000100:2.0:1787828926.850864:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787828926.850892:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828926/real 1787828926] req@ffff8cb999c6bc00 x1874672572571264/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828951 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828926.850904:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c6bc00 x1874672572571264/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828951 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828926.850911:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828926.850914:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787828926.850917:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c6bc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572571264:192.168.203.129@tcp:38:kworker.0 00000100:00100000:2.0:1787828926.850922:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c68f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572571392:192.168.203.129@tcp:38:kworker.0 00000800:00000100:2.0:1787828926.850937:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787828926.850945:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828926/real 1787828926] req@ffff8cb999c68f00 x1874672572571392/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828951 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828926.850955:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c68f00 x1874672572571392/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828951 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828926.850993:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828926.850997:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787828926.851002:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c68f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572571392:192.168.203.129@tcp:38:kworker.0 00010000:00010000:3.0:1787828927.682362:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787828928.706412:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787828929.730372:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787828930.754383:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787828931.778420:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787828931.970371:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828931.970390:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787828931.970393:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828931.970401:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 1884 00000100:00080000:3.0:1787828931.970411:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828931.970417:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:3.0:1787828931.970462:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb992317480 x1874672572571520/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828931.970493:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828931.970497:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787828931.970514:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828931.970518:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 1884 00000100:00080000:3.0:1787828931.970522:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828931.970525:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:2.0:1787828931.970531:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb992317480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572571520:192.168.203.129@tcp:250:kworker.0 00000008:00100000:3.0:1787828931.970533:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787828931.970551:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb992315e00 x1874672572571648/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828931.970564:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828931.970568:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1787828931.970570:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828931.970575:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 1884 00000100:00080000:3.0:1787828931.970579:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828931.970583:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000800:00000100:2.0:1787828931.970589:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000008:00100000:3.0:1787828931.970592:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787828931.970605:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb992316940 x1874672572571776/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787828931.970642:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828931/real 1787828931] req@ffff8cb992317480 x1874672572571520/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828956 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828931.970665:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb992317480 x1874672572571520/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828956 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828931.970674:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828931.970678:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787828931.970681:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb992317480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572571520:192.168.203.129@tcp:250:kworker.0 00000100:00100000:2.0:1787828931.970702:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb992315e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572571648:192.168.203.129@tcp:38:kworker.0 00000800:00000100:2.0:1787828931.970720:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787828931.970729:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828931/real 1787828931] req@ffff8cb992315e00 x1874672572571648/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828956 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828931.970738:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb992315e00 x1874672572571648/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828956 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828931.970754:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828931.970756:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787828931.970759:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb992315e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572571648:192.168.203.129@tcp:38:kworker.0 00000100:00100000:2.0:1787828931.970763:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb992316940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572571776:192.168.203.129@tcp:38:kworker.0 00000800:00000100:2.0:1787828931.970778:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787828931.970785:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828931/real 1787828931] req@ffff8cb992316940 x1874672572571776/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828956 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828931.970792:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb992316940 x1874672572571776/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828956 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828931.970797:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828931.970799:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787828931.970801:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb992316940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572571776:192.168.203.129@tcp:38:kworker.0 00010000:00010000:2.0:1787828932.802404:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787828933.826368:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787828934.850490:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787828935.874429:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787828936.898418:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787828937.090379:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828937.090391:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787828937.090396:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828937.090407:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 1889 00000100:00080000:3.0:1787828937.090421:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828937.090428:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:3.0:1787828937.090500:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb9923161c0 x1874672572571904/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828937.090536:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828937.090539:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787828937.090541:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828937.090544:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 1889 00000100:00080000:3.0:1787828937.090547:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828937.090551:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:3.0:1787828937.090578:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787828937.090590:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb992315a40 x1874672572572032/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828937.090598:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828937.090601:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1787828937.090603:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828937.090606:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 1889 00000100:00080000:3.0:1787828937.090608:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828937.090611:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:3.0:1787828937.090616:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787828937.090623:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb992314f00 x1874672572572160/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787828937.090638:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb9923161c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572571904:192.168.203.129@tcp:250:kworker.0 00000800:00000100:2.0:1787828937.090729:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787828937.090782:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828937/real 1787828937] req@ffff8cb9923161c0 x1874672572571904/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828962 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828937.090800:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb9923161c0 x1874672572571904/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828962 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828937.090832:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828937.090836:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787828937.090840:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb9923161c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572571904:192.168.203.129@tcp:250:kworker.0 00000100:00100000:2.0:1787828937.090848:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb992315a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572572032:192.168.203.129@tcp:38:kworker.0 00000800:00000100:2.0:1787828937.090873:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787828937.090885:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828937/real 1787828937] req@ffff8cb992315a40 x1874672572572032/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828962 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828937.090896:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb992315a40 x1874672572572032/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828962 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828937.090903:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828937.090906:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787828937.090909:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb992315a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572572032:192.168.203.129@tcp:38:kworker.0 00000100:00100000:2.0:1787828937.090934:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb992314f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572572160:192.168.203.129@tcp:38:kworker.0 00000800:00000100:2.0:1787828937.090948:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787828937.090956:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828937/real 1787828937] req@ffff8cb992314f00 x1874672572572160/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828962 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828937.090966:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb992314f00 x1874672572572160/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828962 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828937.090973:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828937.090976:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787828937.090978:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb992314f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572572160:192.168.203.129@tcp:38:kworker.0 00010000:00010000:2.0:1787828937.922415:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787828938.946461:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787828939.970468:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787828940.994537:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787828942.018494:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787828942.210387:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828942.210400:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787828942.210430:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828942.210443:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 1894 00000100:00080000:3.0:1787828942.210456:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828942.210465:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:3.0:1787828942.210519:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb9923170c0 x1874672572572288/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828942.210574:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828942.210581:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787828942.210584:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828942.210590:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 1894 00000100:00080000:3.0:1787828942.210596:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828942.210602:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:3.0:1787828942.210616:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787828942.210639:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb992314f00 x1874672572572416/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787828942.210656:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb9923170c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572572288:192.168.203.129@tcp:250:kworker.0 00000100:00080000:3.0:1787828942.210660:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828942.210673:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1787828942.210676:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828942.210682:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 1894 00000100:00080000:3.0:1787828942.210686:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828942.210691:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:3.0:1787828942.210698:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787828942.210716:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb992315a40 x1874672572572544/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1787828942.210757:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787828942.210845:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828942/real 1787828942] req@ffff8cb9923170c0 x1874672572572288/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828967 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828942.210869:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb9923170c0 x1874672572572288/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828967 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828942.210884:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828942.210907:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787828942.210912:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb9923170c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572572288:192.168.203.129@tcp:250:kworker.0 00000100:00100000:2.0:1787828942.210948:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb992314f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572572416:192.168.203.129@tcp:38:kworker.0 00000800:00000100:2.0:1787828942.210980:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787828942.210997:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828942/real 1787828942] req@ffff8cb992314f00 x1874672572572416/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828967 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828942.211043:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb992314f00 x1874672572572416/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828967 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828942.211084:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828942.211088:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787828942.211093:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb992314f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572572416:192.168.203.129@tcp:38:kworker.0 00000100:00100000:2.0:1787828942.211116:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb992315a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572572544:192.168.203.129@tcp:38:kworker.0 00000800:00000100:2.0:1787828942.211146:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787828942.211161:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828942/real 1787828942] req@ffff8cb992315a40 x1874672572572544/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828967 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828942.211176:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb992315a40 x1874672572572544/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828967 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828942.211185:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828942.211189:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787828942.211193:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb992315a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572572544:192.168.203.129@tcp:38:kworker.0 00010000:00010000:3.0:1787828943.042394:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787828944.066486:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787828945.090433:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787828946.114412:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787828947.138452:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787828947.330502:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828947.330512:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787828947.330516:0:37:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828947.330526:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 1899 00000100:00080000:2.0:1787828947.330537:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828947.330543:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:2.0:1787828947.330581:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8da40 x1874672572572672/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828947.330612:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828947.330616:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787828947.330618:0:37:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828947.330652:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 1899 00000100:00080000:2.0:1787828947.330658:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828947.330663:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:2.0:1787828947.330676:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787828947.330695:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8e1c0 x1874672572572800/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828947.330713:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1787828947.330720:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8da40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572572672:192.168.203.129@tcp:250:kworker.0 00000100:00080000:2.0:1787828947.330722:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1787828947.330729:0:37:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828947.330738:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 1899 00000100:00080000:2.0:1787828947.330743:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828947.330749:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:2.0:1787828947.330756:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787828947.330773:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8fc00 x1874672572572928/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1787828947.330792:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787828947.330836:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828947/real 1787828947] req@ffff8cb998a8da40 x1874672572572672/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828972 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828947.330853:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8da40 x1874672572572672/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828972 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828947.330864:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828947.330868:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787828947.330872:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8da40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572572672:192.168.203.129@tcp:250:kworker.0 00000100:00100000:1.0:1787828947.330910:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8e1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572572800:192.168.203.129@tcp:38:kworker.0 00000800:00000100:1.0:1787828947.330940:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787828947.330959:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828947/real 1787828947] req@ffff8cb998a8e1c0 x1874672572572800/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828972 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828947.331005:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8e1c0 x1874672572572800/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828972 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828947.331034:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828947.331039:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787828947.331044:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8e1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572572800:192.168.203.129@tcp:38:kworker.0 00000100:00100000:1.0:1787828947.331063:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8fc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572572928:192.168.203.129@tcp:38:kworker.0 00000800:00000100:1.0:1787828947.331107:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787828947.331120:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828947/real 1787828947] req@ffff8cb998a8fc00 x1874672572572928/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828972 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828947.331132:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8fc00 x1874672572572928/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828972 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828947.331140:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828947.331144:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787828947.331147:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8fc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572572928:192.168.203.129@tcp:38:kworker.0 00010000:00010000:2.0:1787828948.162582:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787828949.186411:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787828950.210428:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787828951.234472:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787828952.258360:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787828952.450386:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828952.450395:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787828952.450399:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828952.450407:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 1904 00000100:00080000:3.0:1787828952.450418:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828952.450424:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:3.0:1787828952.450459:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb9923161c0 x1874672572573056/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828952.450484:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828952.450488:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787828952.450490:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828952.450494:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 1904 00000100:00080000:3.0:1787828952.450499:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828952.450504:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:3.0:1787828952.450514:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787828952.450516:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb9923161c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572573056:192.168.203.129@tcp:250:kworker.0 00000100:00100000:3.0:1787828952.450531:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb992316940 x1874672572573184/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828952.450544:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828952.450548:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1787828952.450565:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:1.0:1787828952.450572:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00080000:3.0:1787828952.450573:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 1904 00000100:00080000:3.0:1787828952.450591:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828952.450601:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:1.0:1787828952.450608:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828952/real 1787828952] req@ffff8cb9923161c0 x1874672572573056/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828977 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:3.0:1787828952.450609:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787828952.450627:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb992315e00 x1874672572573312/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787828952.450635:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb9923161c0 x1874672572573056/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828977 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828952.450645:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828952.450648:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787828952.450651:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb9923161c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572573056:192.168.203.129@tcp:250:kworker.0 00000100:00100000:1.0:1787828952.450669:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb992316940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572573184:192.168.203.129@tcp:38:kworker.0 00000800:00000100:1.0:1787828952.450685:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787828952.450693:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828952/real 1787828952] req@ffff8cb992316940 x1874672572573184/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828977 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828952.450709:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb992316940 x1874672572573184/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828977 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828952.450715:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828952.450717:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787828952.450719:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb992316940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572573184:192.168.203.129@tcp:38:kworker.0 00000100:00100000:1.0:1787828952.450723:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb992315e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572573312:192.168.203.129@tcp:38:kworker.0 00000800:00000100:1.0:1787828952.450744:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787828952.450751:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828952/real 1787828952] req@ffff8cb992315e00 x1874672572573312/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828977 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828952.450759:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb992315e00 x1874672572573312/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828977 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828952.450764:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828952.450767:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787828952.450769:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb992315e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572573312:192.168.203.129@tcp:38:kworker.0 00010000:00010000:2.0:1787828953.282411:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787828954.306453:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787828955.330354:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787828956.354473:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787828957.378410:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787828957.570387:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828957.570399:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787828957.570404:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828957.570417:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 1909 00000100:00080000:3.0:1787828957.570428:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828957.570438:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:3.0:1787828957.570479:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb992317480 x1874672572573440/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828957.570514:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828957.570519:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787828957.570522:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828957.570527:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 1909 00000100:00080000:3.0:1787828957.570540:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828957.570546:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:3.0:1787828957.570574:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787828957.570593:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb992317480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572573440:192.168.203.129@tcp:250:kworker.0 00000100:00100000:3.0:1787828957.570595:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb992314b40 x1874672572573568/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828957.570623:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828957.570628:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1787828957.570630:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828957.570636:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 1909 00000100:00080000:3.0:1787828957.570654:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828957.570661:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:3.0:1787828957.570668:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000800:00000100:2.0:1787828957.570673:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:3.0:1787828957.570689:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb992314780 x1874672572573696/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787828957.570725:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828957/real 1787828957] req@ffff8cb992317480 x1874672572573440/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828982 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828957.570759:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb992317480 x1874672572573440/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828982 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828957.570770:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828957.570774:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787828957.570780:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb992317480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572573440:192.168.203.129@tcp:250:kworker.0 00000100:00100000:2.0:1787828957.570804:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb992314b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572573568:192.168.203.129@tcp:38:kworker.0 00000800:00000100:2.0:1787828957.570834:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787828957.570851:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828957/real 1787828957] req@ffff8cb992314b40 x1874672572573568/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828982 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828957.570867:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb992314b40 x1874672572573568/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828982 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828957.570879:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828957.570883:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787828957.570887:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb992314b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572573568:192.168.203.129@tcp:38:kworker.0 00000100:00100000:2.0:1787828957.570895:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb992314780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572573696:192.168.203.129@tcp:38:kworker.0 00000800:00000100:2.0:1787828957.570933:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787828957.570943:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828957/real 1787828957] req@ffff8cb992314780 x1874672572573696/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828982 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828957.570954:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb992314780 x1874672572573696/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828982 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828957.570972:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828957.570976:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787828957.570979:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb992314780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572573696:192.168.203.129@tcp:38:kworker.0 00010000:00010000:3.0:1787828958.402371:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787828959.426489:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787828960.450397:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787828961.474380:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787828962.498446:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787828962.690354:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828962.690365:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787828962.690370:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828962.690386:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 1914 00000100:00080000:1.0:1787828962.690398:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828962.690415:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:1.0:1787828962.690460:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f38b40 x1874672572573824/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828962.690502:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828962.690505:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787828962.690507:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828962.690510:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 1914 00000100:00080000:1.0:1787828962.690513:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828962.690516:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:1.0:1787828962.690527:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787828962.690536:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f39e00 x1874672572573952/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828962.690545:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828962.690547:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1787828962.690549:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828962.690552:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 1914 00000100:00080000:1.0:1787828962.690554:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828962.690556:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:1.0:1787828962.690560:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787828962.690568:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f3b840 x1874672572574080/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787828962.690609:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f38b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572573824:192.168.203.129@tcp:250:kworker.0 00000800:00000100:2.0:1787828962.690689:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787828962.690750:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828962/real 1787828962] req@ffff8cb999f38b40 x1874672572573824/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828987 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828962.690771:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f38b40 x1874672572573824/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828987 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828962.690783:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828962.690788:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787828962.690792:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f38b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572573824:192.168.203.129@tcp:250:kworker.0 00000100:00100000:2.0:1787828962.690800:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f39e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572573952:192.168.203.129@tcp:38:kworker.0 00000800:00000100:2.0:1787828962.690821:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787828962.690833:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828962/real 1787828962] req@ffff8cb999f39e00 x1874672572573952/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828987 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828962.690845:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f39e00 x1874672572573952/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828987 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828962.690896:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828962.690900:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787828962.690904:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f39e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572573952:192.168.203.129@tcp:38:kworker.0 00000100:00100000:2.0:1787828962.690911:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f3b840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572574080:192.168.203.129@tcp:38:kworker.0 00000800:00000100:2.0:1787828962.690934:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787828962.690946:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828962/real 1787828962] req@ffff8cb999f3b840 x1874672572574080/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828987 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828962.690957:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f3b840 x1874672572574080/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828987 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828962.690965:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828962.690967:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787828962.690970:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f3b840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572574080:192.168.203.129@tcp:38:kworker.0 00010000:00010000:2.0:1787828963.522452:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787828964.546414:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787828965.570485:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787828966.594474:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787828967.618370:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787828967.810378:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828967.810392:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787828967.810397:0:37:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828967.810410:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 1919 00000100:00080000:0.0:1787828967.810444:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828967.810453:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:0.0:1787828967.810498:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c69a40 x1874672572574208/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828967.810546:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828967.810551:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787828967.810554:0:37:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828967.810558:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 1919 00000100:00080000:0.0:1787828967.810562:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828967.810567:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:0.0:1787828967.810579:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787828967.810604:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c6b840 x1874672572574336/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828967.810617:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828967.810622:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1787828967.810624:0:37:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828967.810629:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 1919 00000100:00080000:0.0:1787828967.810634:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828967.810639:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:0.0:1787828967.810646:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787828967.810670:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c69a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572574208:192.168.203.129@tcp:250:kworker.0 00000100:00100000:0.0:1787828967.810678:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c68f00 x1874672572574464/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1787828967.810749:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787828967.810796:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828967/real 1787828967] req@ffff8cb999c69a40 x1874672572574208/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828992 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828967.810815:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c69a40 x1874672572574208/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828992 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828967.810826:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828967.810860:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787828967.810864:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c69a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572574208:192.168.203.129@tcp:250:kworker.0 00000100:00100000:2.0:1787828967.810871:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c6b840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572574336:192.168.203.129@tcp:38:kworker.0 00000800:00000100:2.0:1787828967.810908:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787828967.810918:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828967/real 1787828967] req@ffff8cb999c6b840 x1874672572574336/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828992 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828967.810926:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c6b840 x1874672572574336/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828992 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828967.810932:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828967.810934:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787828967.810937:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c6b840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572574336:192.168.203.129@tcp:38:kworker.0 00000100:00100000:2.0:1787828967.810957:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c68f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572574464:192.168.203.129@tcp:38:kworker.0 00000800:00000100:2.0:1787828967.810968:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787828967.810975:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828967/real 1787828967] req@ffff8cb999c68f00 x1874672572574464/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828992 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828967.810983:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c68f00 x1874672572574464/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828992 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828967.810988:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828967.810989:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787828967.810991:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c68f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572574464:192.168.203.129@tcp:38:kworker.0 00010000:00010000:1.0:1787828968.642451:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787828969.666403:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787828970.690420:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787828971.714432:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787828972.738508:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787828972.930428:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828972.930443:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787828972.930448:0:37:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828972.930460:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 1925 00000100:00080000:2.0:1787828972.930474:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828972.930483:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:2.0:1787828972.930536:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8f0c0 x1874672572574592/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828972.930571:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828972.930576:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787828972.930578:0:37:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828972.930583:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 1925 00000100:00080000:2.0:1787828972.930587:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828972.930592:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:2.0:1787828972.930621:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787828972.930625:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8f0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572574592:192.168.203.129@tcp:250:kworker.0 00000100:00100000:2.0:1787828972.930641:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8c780 x1874672572574720/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828972.930657:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828972.930661:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1787828972.930664:0:37:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828972.930669:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 1925 00000100:00080000:2.0:1787828972.930675:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828972.930681:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000800:00000100:1.0:1787828972.930707:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000008:00100000:2.0:1787828972.930711:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787828972.930733:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8cf00 x1874672572574848/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787828972.930781:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828972/real 1787828972] req@ffff8cb998a8f0c0 x1874672572574592/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828997 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828972.930826:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8f0c0 x1874672572574592/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828997 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828972.930857:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828972.930862:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787828972.930867:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8f0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572574592:192.168.203.129@tcp:250:kworker.0 00000100:00100000:1.0:1787828972.930896:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8c780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572574720:192.168.203.129@tcp:38:kworker.0 00000800:00000100:1.0:1787828972.930923:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787828972.930939:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828972/real 1787828972] req@ffff8cb998a8c780 x1874672572574720/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828997 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828972.930953:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8c780 x1874672572574720/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828997 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828972.930962:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828972.930966:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787828972.930970:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8c780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572574720:192.168.203.129@tcp:38:kworker.0 00000100:00100000:1.0:1787828972.930976:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8cf00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572574848:192.168.203.129@tcp:38:kworker.0 00000800:00000100:1.0:1787828972.931012:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787828972.931022:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828972/real 1787828972] req@ffff8cb998a8cf00 x1874672572574848/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828997 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828972.931032:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8cf00 x1874672572574848/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828997 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828972.931050:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828972.931053:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787828972.931056:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8cf00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572574848:192.168.203.129@tcp:38:kworker.0 00010000:00010000:3.0:1787828973.762376:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787828974.786413:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787828975.810453:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787828976.834381:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787828977.858418:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787828978.050417:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828978.050427:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787828978.050432:0:37:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828978.050443:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 1930 00000100:00080000:0.0:1787828978.050456:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828978.050463:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:0.0:1787828978.050514:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c6bc00 x1874672572574976/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828978.050539:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828978.050542:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787828978.050544:0:37:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828978.050546:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 1930 00000100:00080000:0.0:1787828978.050548:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828978.050551:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:0.0:1787828978.050560:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787828978.050568:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c69e00 x1874672572575104/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828978.050575:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828978.050577:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1787828978.050578:0:37:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828978.050581:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 1930 00000100:00080000:0.0:1787828978.050583:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828978.050586:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:0.0:1787828978.050608:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787828978.050616:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c6bc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572574976:192.168.203.129@tcp:250:kworker.0 00000100:00100000:0.0:1787828978.050618:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c6b480 x1874672572575232/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787828978.050732:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c69e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572575104:192.168.203.129@tcp:38:kworker.0 00000100:00100000:1.0:1787828978.050777:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c6b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572575232:192.168.203.129@tcp:38:kworker.0 00010000:00010000:1.0:1787828978.882439:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787828979.906388:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787828980.930594:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:3.0:1787828981.122668:0:2381:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.129:988 00000400:00000100:3.0:1787828981.122721:0:2381:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.129@tcp at host 192.168.203.129 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:3.0:1787828981.122758:0:2381:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.129@tcp 00000800:00000100:3.0:1787828981.122812:0:2381:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.129@tcp 00000800:00000100:3.0:1787828981.122823:0:2381:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.129@tcp 00000100:00100000:1.0:1787828981.122867:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828978/real 1787828981] req@ffff8cb999c6b480 x1874672572575232/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829003 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828981.122895:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c6b480 x1874672572575232/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829003 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828981.122908:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828981.122914:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787828981.122919:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c6b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572575232:192.168.203.129@tcp:38:kworker.0 00000100:00100000:1.0:1787828981.122927:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828978/real 1787828981] req@ffff8cb999c6bc00 x1874672572574976/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829003 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828981.122941:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c6bc00 x1874672572574976/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829003 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828981.122949:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828981.122954:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787828981.122976:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c6bc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572574976:192.168.203.129@tcp:250:kworker.0 00000100:00100000:1.0:1787828981.122984:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828978/real 1787828981] req@ffff8cb999c69e00 x1874672572575104/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829003 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828981.122997:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c69e00 x1874672572575104/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829003 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828981.123006:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828981.123009:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787828981.123012:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c69e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572575104:192.168.203.129@tcp:38:kworker.0 00010000:00010000:2.0:1787828981.954582:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787828982.978439:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787828983.170381:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828983.170392:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787828983.170397:0:37:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828983.170409:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 1935 00000100:00080000:0.0:1787828983.170420:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828983.170427:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:0.0:1787828983.170462:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c692c0 x1874672572575360/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828983.170485:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828983.170488:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787828983.170490:0:37:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828983.170495:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 1935 00000100:00080000:0.0:1787828983.170499:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828983.170505:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:0.0:1787828983.170516:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787828983.170525:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c692c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572575360:192.168.203.129@tcp:250:kworker.0 00000100:00100000:0.0:1787828983.170539:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c69680 x1874672572575488/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828983.170555:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828983.170560:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1787828983.170562:0:37:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828983.170567:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 1935 00000100:00080000:0.0:1787828983.170585:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828983.170590:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:0.0:1787828983.170595:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787828983.170605:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c68000 x1874672572575616/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1787828983.170627:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787828983.170691:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828983/real 1787828983] req@ffff8cb999c692c0 x1874672572575360/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829008 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828983.170715:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c692c0 x1874672572575360/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829008 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828983.170745:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828983.170752:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787828983.170756:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c692c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572575360:192.168.203.129@tcp:250:kworker.0 00000100:00100000:1.0:1787828983.170794:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c69680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572575488:192.168.203.129@tcp:38:kworker.0 00000800:00000100:1.0:1787828983.170821:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787828983.170838:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828983/real 1787828983] req@ffff8cb999c69680 x1874672572575488/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829008 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828983.170877:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c69680 x1874672572575488/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829008 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828983.170887:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828983.170891:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787828983.170895:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c69680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572575488:192.168.203.129@tcp:38:kworker.0 00000100:00100000:1.0:1787828983.170903:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c68000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572575616:192.168.203.129@tcp:38:kworker.0 00000800:00000100:1.0:1787828983.170929:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787828983.170944:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828983/real 1787828983] req@ffff8cb999c68000 x1874672572575616/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829008 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828983.170956:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c68000 x1874672572575616/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829008 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828983.170964:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828983.170967:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787828983.170970:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c68000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572575616:192.168.203.129@tcp:38:kworker.0 00010000:00010000:2.0:1787828984.002438:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787828985.026632:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787828986.050462:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787828987.074661:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787828988.098523:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787828988.290399:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828988.290407:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787828988.290410:0:37:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828988.290418:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 1940 00000100:00080000:2.0:1787828988.290428:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828988.290433:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:2.0:1787828988.290464:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8e580 x1874672572575744/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828988.290494:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828988.290497:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787828988.290498:0:37:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828988.290501:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 1940 00000100:00080000:2.0:1787828988.290503:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828988.290506:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:2.0:1787828988.290515:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787828988.290531:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8e580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572575744:192.168.203.129@tcp:250:kworker.0 00000100:00100000:2.0:1787828988.290540:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8d680 x1874672572575872/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828988.290550:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828988.290552:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1787828988.290554:0:37:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828988.290557:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 1940 00000100:00080000:2.0:1787828988.290560:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828988.290563:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:2.0:1787828988.290567:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787828988.290574:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8cb40 x1874672572576000/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1787828988.290588:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787828988.290624:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828988/real 1787828988] req@ffff8cb998a8e580 x1874672572575744/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829013 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828988.290638:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8e580 x1874672572575744/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829013 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828988.290657:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828988.290659:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787828988.290662:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8e580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572575744:192.168.203.129@tcp:250:kworker.0 00000100:00100000:1.0:1787828988.290682:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8d680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572575872:192.168.203.129@tcp:38:kworker.0 00000800:00000100:1.0:1787828988.290720:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787828988.290730:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828988/real 1787828988] req@ffff8cb998a8d680 x1874672572575872/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829013 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828988.290739:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8d680 x1874672572575872/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829013 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828988.290745:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828988.290747:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787828988.290749:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8d680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572575872:192.168.203.129@tcp:38:kworker.0 00000100:00100000:1.0:1787828988.290753:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8cb40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572576000:192.168.203.129@tcp:38:kworker.0 00000800:00000100:1.0:1787828988.290769:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787828988.290780:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828988/real 1787828988] req@ffff8cb998a8cb40 x1874672572576000/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829013 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828988.290792:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8cb40 x1874672572576000/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829013 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828988.290803:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828988.290806:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787828988.290810:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8cb40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572576000:192.168.203.129@tcp:38:kworker.0 00010000:00010000:3.0:1787828989.122697:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787828990.147204:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787828991.170467:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787828992.194415:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787828993.218474:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787828993.410429:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828993.410435:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787828993.410440:0:37:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828993.410451:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 1945 00000100:00080000:0.0:1787828993.410462:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828993.410471:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:0.0:1787828993.410505:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c6a580 x1874672572576128/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828993.410539:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828993.410543:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787828993.410545:0:37:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828993.410549:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 1945 00000100:00080000:0.0:1787828993.410554:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828993.410559:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:0.0:1787828993.410569:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787828993.410587:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c6a1c0 x1874672572576256/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787828993.410593:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c6a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572576128:192.168.203.129@tcp:250:kworker.0 00000100:00080000:0.0:1787828993.410604:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828993.410614:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1787828993.410616:0:37:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828993.410621:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 1945 00000100:00080000:0.0:1787828993.410628:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828993.410631:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:0.0:1787828993.410635:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787828993.410650:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c68000 x1874672572576384/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1787828993.410700:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787828993.410752:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828993/real 1787828993] req@ffff8cb999c6a580 x1874672572576128/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829018 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828993.410769:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c6a580 x1874672572576128/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829018 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828993.410780:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828993.410784:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787828993.410788:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c6a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572576128:192.168.203.129@tcp:250:kworker.0 00000100:00100000:2.0:1787828993.410836:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c6a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572576256:192.168.203.129@tcp:38:kworker.0 00000800:00000100:2.0:1787828993.410856:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787828993.410867:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828993/real 1787828993] req@ffff8cb999c6a1c0 x1874672572576256/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829018 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828993.410875:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c6a1c0 x1874672572576256/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829018 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828993.410881:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828993.410885:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787828993.410888:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c6a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572576256:192.168.203.129@tcp:38:kworker.0 00000100:00100000:2.0:1787828993.410892:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c68000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572576384:192.168.203.129@tcp:38:kworker.0 00000800:00000100:2.0:1787828993.410905:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787828993.410911:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828993/real 1787828993] req@ffff8cb999c68000 x1874672572576384/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829018 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828993.410920:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c68000 x1874672572576384/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829018 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828993.410925:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828993.410928:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787828993.410930:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c68000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572576384:192.168.203.129@tcp:38:kworker.0 00010000:00010000:2.0:1787828994.242423:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787828995.266399:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787828996.290419:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787828997.314442:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787828998.338499:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787828998.530400:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828998.530411:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787828998.530415:0:37:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828998.530427:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 1950 00000100:00080000:0.0:1787828998.530437:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828998.530444:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:0.0:1787828998.530502:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c69680 x1874672572576512/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828998.530536:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828998.530540:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787828998.530542:0:37:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828998.530546:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 1950 00000100:00080000:0.0:1787828998.530550:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828998.530555:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:0.0:1787828998.530568:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787828998.530583:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c69680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572576512:192.168.203.129@tcp:250:kworker.0 00000100:00100000:0.0:1787828998.530586:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c692c0 x1874672572576640/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828998.530632:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828998.530636:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1787828998.530637:0:37:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828998.530642:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 1950 00000100:00080000:0.0:1787828998.530646:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828998.530651:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:0.0:1787828998.530658:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787828998.530673:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c69e00 x1874672572576768/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1787828998.530711:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787828998.530764:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828998/real 1787828998] req@ffff8cb999c69680 x1874672572576512/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829023 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828998.530786:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c69680 x1874672572576512/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829023 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828998.530798:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828998.530803:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787828998.530809:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c69680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572576512:192.168.203.129@tcp:250:kworker.0 00000100:00100000:2.0:1787828998.530835:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c692c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572576640:192.168.203.129@tcp:38:kworker.0 00000800:00000100:2.0:1787828998.530854:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787828998.530865:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828998/real 1787828998] req@ffff8cb999c692c0 x1874672572576640/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829023 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828998.530873:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c692c0 x1874672572576640/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829023 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828998.530879:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828998.530882:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787828998.530884:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c692c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572576640:192.168.203.129@tcp:38:kworker.0 00000100:00100000:2.0:1787828998.530888:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c69e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572576768:192.168.203.129@tcp:38:kworker.0 00000800:00000100:2.0:1787828998.530899:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787828998.530906:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828998/real 1787828998] req@ffff8cb999c69e00 x1874672572576768/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829023 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828998.530924:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c69e00 x1874672572576768/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829023 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828998.530930:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828998.530932:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787828998.530934:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c69e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572576768:192.168.203.129@tcp:38:kworker.0 00010000:00010000:2.0:1787828999.362427:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829000.386442:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787829001.410388:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829002.434450:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829003.458498:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787829003.650381:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829003.650391:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787829003.650395:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829003.650405:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 1955 00000100:00080000:1.0:1787829003.650417:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829003.650424:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:1.0:1787829003.650459:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f39a40 x1874672572576896/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829003.650482:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829003.650485:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787829003.650487:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829003.650490:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 1955 00000100:00080000:1.0:1787829003.650493:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829003.650495:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:1.0:1787829003.650504:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787829003.650514:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f3b840 x1874672572577024/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829003.650522:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829003.650527:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00100000:2.0:1787829003.650527:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f39a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572576896:192.168.203.129@tcp:250:kworker.0 00000100:00080000:1.0:1787829003.650530:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829003.650537:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 1955 00000100:00080000:1.0:1787829003.650540:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829003.650542:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:1.0:1787829003.650559:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787829003.650569:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f39e00 x1874672572577152/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1787829003.650606:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787829003.650662:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829003/real 1787829003] req@ffff8cb999f39a40 x1874672572576896/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829028 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829003.650677:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f39a40 x1874672572576896/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829028 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829003.650686:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829003.650688:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787829003.650691:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f39a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572576896:192.168.203.129@tcp:250:kworker.0 00000100:00100000:2.0:1787829003.650711:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f3b840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572577024:192.168.203.129@tcp:38:kworker.0 00000800:00000100:2.0:1787829003.650729:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787829003.650738:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829003/real 1787829003] req@ffff8cb999f3b840 x1874672572577024/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829028 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829003.650758:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f3b840 x1874672572577024/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829028 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829003.650764:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829003.650766:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787829003.650768:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f3b840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572577024:192.168.203.129@tcp:38:kworker.0 00000100:00100000:2.0:1787829003.650773:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f39e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572577152:192.168.203.129@tcp:38:kworker.0 00000800:00000100:2.0:1787829003.650786:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787829003.650793:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829003/real 1787829003] req@ffff8cb999f39e00 x1874672572577152/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829028 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829003.650801:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f39e00 x1874672572577152/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829028 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829003.650806:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829003.650808:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787829003.650810:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f39e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572577152:192.168.203.129@tcp:38:kworker.0 00010000:00010000:3.0:1787829004.482449:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829005.506381:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829006.530419:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787829007.554368:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829008.578533:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787829008.770355:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829008.770367:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787829008.770372:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829008.770382:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 1960 00000100:00080000:0.0:1787829008.770394:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829008.770400:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:0.0:1787829008.770442:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c6bc00 x1874672572577280/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829008.770467:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829008.770470:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787829008.770472:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829008.770474:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 1960 00000100:00080000:0.0:1787829008.770477:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829008.770479:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:0.0:1787829008.770502:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787829008.770513:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c6b480 x1874672572577408/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829008.770520:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829008.770523:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1787829008.770525:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829008.770528:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 1960 00000100:00080000:0.0:1787829008.770531:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829008.770535:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:0.0:1787829008.770539:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787829008.770555:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c6bc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572577280:192.168.203.129@tcp:250:kworker.0 00000100:00100000:0.0:1787829008.770566:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c68f00 x1874672572577536/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1787829008.770625:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787829008.770662:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829008/real 1787829008] req@ffff8cb999c6bc00 x1874672572577280/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829033 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829008.770691:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c6bc00 x1874672572577280/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829033 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829008.770699:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829008.770702:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787829008.770705:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c6bc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572577280:192.168.203.129@tcp:250:kworker.0 00000100:00100000:2.0:1787829008.770711:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c6b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572577408:192.168.203.129@tcp:38:kworker.0 00000800:00000100:2.0:1787829008.770727:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787829008.770735:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829008/real 1787829008] req@ffff8cb999c6b480 x1874672572577408/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829033 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829008.770743:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c6b480 x1874672572577408/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829033 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829008.770749:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829008.770751:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787829008.770753:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c6b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572577408:192.168.203.129@tcp:38:kworker.0 00000100:00100000:2.0:1787829008.770774:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c68f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572577536:192.168.203.129@tcp:38:kworker.0 00000800:00000100:2.0:1787829008.770786:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787829008.770792:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829008/real 1787829008] req@ffff8cb999c68f00 x1874672572577536/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829033 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829008.770799:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c68f00 x1874672572577536/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829033 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829008.770814:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829008.770817:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787829008.770819:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c68f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572577536:192.168.203.129@tcp:38:kworker.0 00010000:00010000:2.0:1787829009.602433:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787829010.626415:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829011.650396:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829012.674444:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787829013.698455:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787829013.890499:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829013.890529:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787829013.890534:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829013.890546:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 1966 00000100:00080000:1.0:1787829013.890577:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829013.890584:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:1.0:1787829013.890621:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f38b40 x1874672572577664/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829013.890645:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829013.890649:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787829013.890652:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829013.890656:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 1966 00000100:00080000:1.0:1787829013.890660:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829013.890663:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:1.0:1787829013.890673:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787829013.890686:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f3b480 x1874672572577792/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787829013.890700:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f38b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572577664:192.168.203.129@tcp:250:kworker.0 00000100:00080000:1.0:1787829013.890701:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829013.890720:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1787829013.890723:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829013.890729:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 1966 00000100:00080000:1.0:1787829013.890752:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829013.890759:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:1.0:1787829013.890769:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787829013.890791:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f3b0c0 x1874672572577920/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1787829013.890813:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787829013.890877:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829013/real 1787829013] req@ffff8cb999f38b40 x1874672572577664/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829038 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829013.890899:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f38b40 x1874672572577664/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829038 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829013.890909:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829013.890912:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787829013.890916:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f38b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572577664:192.168.203.129@tcp:250:kworker.0 00000100:00100000:2.0:1787829013.890944:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f3b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572577792:192.168.203.129@tcp:38:kworker.0 00000800:00000100:2.0:1787829013.890960:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787829013.890969:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829013/real 1787829013] req@ffff8cb999f3b480 x1874672572577792/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829038 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829013.890977:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f3b480 x1874672572577792/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829038 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829013.890999:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829013.891002:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787829013.891004:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f3b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572577792:192.168.203.129@tcp:38:kworker.0 00000100:00100000:2.0:1787829013.891009:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f3b0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572577920:192.168.203.129@tcp:38:kworker.0 00000800:00000100:2.0:1787829013.891023:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787829013.891031:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829013/real 1787829013] req@ffff8cb999f3b0c0 x1874672572577920/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829038 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829013.891038:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f3b0c0 x1874672572577920/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829038 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829013.891043:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829013.891045:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787829013.891047:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f3b0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572577920:192.168.203.129@tcp:38:kworker.0 00010000:00010000:2.0:1787829014.722581:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829015.746538:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787829016.770396:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829017.794510:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829018.818507:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787829019.010349:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829019.010359:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787829019.010364:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829019.010375:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 1971 00000100:00080000:1.0:1787829019.010387:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829019.010394:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:1.0:1787829019.010434:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f38f00 x1874672572578048/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829019.010467:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829019.010472:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787829019.010493:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829019.010498:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 1971 00000100:00080000:1.0:1787829019.010502:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829019.010507:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:1.0:1787829019.010522:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787829019.010541:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f392c0 x1874672572578176/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829019.010554:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829019.010576:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1787829019.010580:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829019.010586:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 1971 00000100:00080000:1.0:1787829019.010591:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829019.010595:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:1.0:1787829019.010601:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787829019.010615:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f38f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572578048:192.168.203.129@tcp:250:kworker.0 00000100:00100000:1.0:1787829019.010621:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f3a580 x1874672572578304/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1787829019.010771:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787829019.010865:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829019/real 1787829019] req@ffff8cb999f38f00 x1874672572578048/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829044 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829019.010910:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f38f00 x1874672572578048/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829044 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829019.010957:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829019.010966:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787829019.010973:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f38f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572578048:192.168.203.129@tcp:250:kworker.0 00000100:00100000:2.0:1787829019.010987:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f392c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572578176:192.168.203.129@tcp:38:kworker.0 00000800:00000100:2.0:1787829019.011032:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787829019.011071:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829019/real 1787829019] req@ffff8cb999f392c0 x1874672572578176/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829044 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829019.011093:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f392c0 x1874672572578176/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829044 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829019.011106:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829019.011112:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787829019.011117:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f392c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572578176:192.168.203.129@tcp:38:kworker.0 00000100:00100000:2.0:1787829019.011164:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f3a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572578304:192.168.203.129@tcp:38:kworker.0 00000800:00000100:2.0:1787829019.011201:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787829019.011234:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829019/real 1787829019] req@ffff8cb999f3a580 x1874672572578304/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829044 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829019.011256:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f3a580 x1874672572578304/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829044 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829019.011326:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829019.011333:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787829019.011339:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f3a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572578304:192.168.203.129@tcp:38:kworker.0 00010000:00010000:3.0:1787829019.842340:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829020.866386:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829021.890372:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787829022.914404:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829023.938375:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787829024.130378:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829024.130388:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787829024.130392:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829024.130401:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 1976 00000100:00080000:1.0:1787829024.130412:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829024.130419:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:1.0:1787829024.130477:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f383c0 x1874672572578432/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829024.130505:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829024.130509:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787829024.130511:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829024.130514:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 1976 00000100:00080000:1.0:1787829024.130517:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829024.130521:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:1.0:1787829024.130530:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787829024.130547:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f3a580 x1874672572578560/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787829024.130557:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f383c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572578432:192.168.203.129@tcp:250:kworker.0 00000100:00080000:1.0:1787829024.130585:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829024.130590:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1787829024.130605:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829024.130612:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 1976 00000100:00080000:1.0:1787829024.130622:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829024.130629:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000800:00000100:2.0:1787829024.130634:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000008:00100000:1.0:1787829024.130639:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787829024.130658:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f392c0 x1874672572578688/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787829024.130680:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829024/real 1787829024] req@ffff8cb999f383c0 x1874672572578432/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829049 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829024.130698:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f383c0 x1874672572578432/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829049 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829024.130710:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829024.130714:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787829024.130718:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f383c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572578432:192.168.203.129@tcp:250:kworker.0 00000100:00100000:2.0:1787829024.130741:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f3a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572578560:192.168.203.129@tcp:38:kworker.0 00000800:00000100:2.0:1787829024.130760:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787829024.130771:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829024/real 1787829024] req@ffff8cb999f3a580 x1874672572578560/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829049 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829024.130781:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f3a580 x1874672572578560/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829049 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829024.130788:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829024.130791:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787829024.130794:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f3a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572578560:192.168.203.129@tcp:38:kworker.0 00000100:00100000:2.0:1787829024.130799:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f392c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572578688:192.168.203.129@tcp:38:kworker.0 00000800:00000100:2.0:1787829024.130825:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787829024.130834:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829024/real 1787829024] req@ffff8cb999f392c0 x1874672572578688/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829049 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829024.130844:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f392c0 x1874672572578688/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829049 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829024.130851:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829024.130853:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787829024.130856:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f392c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572578688:192.168.203.129@tcp:38:kworker.0 00010000:00010000:2.0:1787829024.962486:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787829025.986463:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829027.010411:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829028.034486:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787829029.058423:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787829029.250361:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829029.250372:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787829029.250377:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829029.250388:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 1981 00000100:00080000:1.0:1787829029.250399:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829029.250418:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:1.0:1787829029.250463:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f38f00 x1874672572578816/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829029.250490:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829029.250493:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787829029.250495:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829029.250497:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 1981 00000100:00080000:1.0:1787829029.250500:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829029.250503:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:1.0:1787829029.250532:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787829029.250543:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f3b0c0 x1874672572578944/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787829029.250555:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f38f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572578816:192.168.203.129@tcp:250:kworker.0 00000100:00080000:1.0:1787829029.250557:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829029.250563:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1787829029.250565:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829029.250575:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 1981 00000100:00080000:1.0:1787829029.250579:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829029.250581:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:1.0:1787829029.250585:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787829029.250602:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f3b480 x1874672572579072/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1787829029.250642:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787829029.250688:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829029/real 1787829029] req@ffff8cb999f38f00 x1874672572578816/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829054 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829029.250703:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f38f00 x1874672572578816/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829054 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829029.250711:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829029.250715:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787829029.250719:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f38f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572578816:192.168.203.129@tcp:250:kworker.0 00000100:00100000:2.0:1787829029.250746:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f3b0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572578944:192.168.203.129@tcp:38:kworker.0 00000800:00000100:2.0:1787829029.250771:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787829029.250804:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829029/real 1787829029] req@ffff8cb999f3b0c0 x1874672572578944/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829054 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829029.250825:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f3b0c0 x1874672572578944/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829054 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829029.250834:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829029.250837:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787829029.250841:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f3b0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572578944:192.168.203.129@tcp:38:kworker.0 00000100:00100000:2.0:1787829029.250852:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f3b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572579072:192.168.203.129@tcp:38:kworker.0 00000800:00000100:2.0:1787829029.250874:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787829029.250883:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829029/real 1787829029] req@ffff8cb999f3b480 x1874672572579072/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829054 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829029.250892:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f3b480 x1874672572579072/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829054 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829029.250897:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829029.250900:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787829029.250910:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f3b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572579072:192.168.203.129@tcp:38:kworker.0 00010000:00010000:2.0:1787829030.082469:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829031.106484:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787829032.130424:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829033.154401:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829034.178403:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787829034.370345:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829034.370353:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787829034.370356:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829034.370365:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 1986 00000100:00080000:3.0:1787829034.370389:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829034.370395:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:3.0:1787829034.370432:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb992316580 x1874672572579200/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787829034.370452:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829034.370454:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787829034.370456:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829034.370458:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 1986 00000100:00080000:3.0:1787829034.370461:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829034.370464:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:3.0:1787829034.370482:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787829034.370494:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb992314780 x1874672572579328/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787829034.370498:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb992316580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572579200:192.168.203.129@tcp:250:kworker.0 00000100:00080000:3.0:1787829034.370505:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829034.370509:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1787829034.370510:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829034.370513:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 1986 00000100:00080000:3.0:1787829034.370516:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829034.370518:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:3.0:1787829034.370521:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787829034.370531:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb992314b40 x1874672572579456/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1787829034.370575:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787829034.370636:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829034/real 1787829034] req@ffff8cb992316580 x1874672572579200/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829059 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829034.370652:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb992316580 x1874672572579200/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829059 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829034.370660:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829034.370663:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787829034.370665:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb992316580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572579200:192.168.203.129@tcp:250:kworker.0 00000100:00100000:2.0:1787829034.370685:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb992314780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572579328:192.168.203.129@tcp:38:kworker.0 00000800:00000100:2.0:1787829034.370704:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787829034.370712:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829034/real 1787829034] req@ffff8cb992314780 x1874672572579328/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829059 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829034.370720:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb992314780 x1874672572579328/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829059 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829034.370726:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829034.370728:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787829034.370730:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb992314780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572579328:192.168.203.129@tcp:38:kworker.0 00000100:00100000:2.0:1787829034.370733:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb992314b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572579456:192.168.203.129@tcp:38:kworker.0 00000800:00000100:2.0:1787829034.370743:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787829034.370749:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829034/real 1787829034] req@ffff8cb992314b40 x1874672572579456/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829059 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829034.370756:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb992314b40 x1874672572579456/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829059 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829034.370771:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829034.370773:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787829034.370776:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb992314b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572579456:192.168.203.129@tcp:38:kworker.0 00010000:00010000:3.0:1787829035.202446:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829036.226441:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829037.250410:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787829038.274402:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829039.298401:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787829039.490351:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829039.490360:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787829039.490363:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829039.490371:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 1991 00000100:00080000:3.0:1787829039.490379:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829039.490385:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:3.0:1787829039.490451:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb992317480 x1874672572579584/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787829039.490478:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829039.490481:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787829039.490509:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829039.490513:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 1991 00000100:00100000:2.0:1787829039.490519:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb992317480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572579584:192.168.203.129@tcp:250:kworker.0 00000100:00080000:3.0:1787829039.490519:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829039.490527:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:3.0:1787829039.490536:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787829039.490554:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb992315e00 x1874672572579712/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787829039.490564:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829039.490566:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1787829039.490568:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829039.490570:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 1991 00000100:00080000:3.0:1787829039.490574:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829039.490576:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:3.0:1787829039.490579:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000800:00000100:2.0:1787829039.490585:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:3.0:1787829039.490589:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb992316940 x1874672572579840/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787829039.490626:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829039/real 1787829039] req@ffff8cb992317480 x1874672572579584/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829064 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829039.490639:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb992317480 x1874672572579584/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829064 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829039.490646:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829039.490649:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787829039.490652:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb992317480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572579584:192.168.203.129@tcp:250:kworker.0 00000100:00100000:2.0:1787829039.490670:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb992315e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572579712:192.168.203.129@tcp:38:kworker.0 00000800:00000100:2.0:1787829039.490686:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787829039.490694:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829039/real 1787829039] req@ffff8cb992315e00 x1874672572579712/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829064 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829039.490702:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb992315e00 x1874672572579712/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829064 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829039.490717:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829039.490720:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787829039.490722:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb992315e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572579712:192.168.203.129@tcp:38:kworker.0 00000100:00100000:2.0:1787829039.490726:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb992316940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572579840:192.168.203.129@tcp:38:kworker.0 00000800:00000100:2.0:1787829039.490741:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787829039.490748:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829039/real 1787829039] req@ffff8cb992316940 x1874672572579840/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829064 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829039.490755:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb992316940 x1874672572579840/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829064 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829039.490761:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829039.490762:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787829039.490764:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb992316940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572579840:192.168.203.129@tcp:38:kworker.0 00010000:00010000:2.0:1787829040.322418:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787829041.346369:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829042.370414:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829043.394477:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787829044.418393:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787829044.610337:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829044.610348:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787829044.610352:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829044.610362:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 1996 00000100:00080000:3.0:1787829044.610373:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829044.610379:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:3.0:1787829044.610427:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb9923161c0 x1874672572579968/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787829044.610453:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829044.610457:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787829044.610459:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829044.610463:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 1996 00000100:00080000:3.0:1787829044.610467:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829044.610471:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:3.0:1787829044.610481:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787829044.610494:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb992315a40 x1874672572580096/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787829044.610503:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb9923161c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572579968:192.168.203.129@tcp:250:kworker.0 00000100:00080000:3.0:1787829044.610508:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829044.610514:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1787829044.610516:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829044.610521:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 1996 00000100:00080000:3.0:1787829044.610525:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829044.610530:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:3.0:1787829044.610535:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787829044.610548:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb992314f00 x1874672572580224/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787829044.610611:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb992315a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572580096:192.168.203.129@tcp:38:kworker.0 00000100:00100000:2.0:1787829044.610644:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb992314f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572580224:192.168.203.129@tcp:38:kworker.0 00010000:00010000:2.0:1787829045.442497:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829046.466416:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787829047.490649:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:0.0:1787829047.682507:0:2382:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.129:988 00000400:00000100:0.0:1787829047.682561:0:2382:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.129@tcp at host 192.168.203.129 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:0.0:1787829047.682578:0:2382:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.129@tcp 00000800:00000100:0.0:1787829047.682619:0:2382:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.129@tcp 00000800:00000100:0.0:1787829047.682631:0:2382:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.129@tcp 00000100:00100000:2.0:1787829047.682662:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829044/real 1787829047] req@ffff8cb992315a40 x1874672572580096/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829069 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829047.682703:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb992315a40 x1874672572580096/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829069 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829047.682716:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829047.682721:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787829047.682726:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb992315a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572580096:192.168.203.129@tcp:38:kworker.0 00000100:00100000:2.0:1787829047.682735:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829044/real 1787829047] req@ffff8cb992314f00 x1874672572580224/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829069 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829047.682748:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb992314f00 x1874672572580224/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829069 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829047.682757:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829047.682760:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787829047.682762:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb992314f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572580224:192.168.203.129@tcp:38:kworker.0 00000100:00100000:2.0:1787829047.682769:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829044/real 1787829047] req@ffff8cb9923161c0 x1874672572579968/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829069 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829047.682783:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb9923161c0 x1874672572579968/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829069 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829047.682791:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829047.682794:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787829047.682798:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb9923161c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572579968:192.168.203.129@tcp:250:kworker.0 00010000:00010000:0.0:1787829048.515110:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787829049.538651:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787829049.730357:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829049.730367:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787829049.730390:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829049.730399:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 2001 00000100:00080000:3.0:1787829049.730410:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829049.730418:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:3.0:1787829049.730453:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb9923170c0 x1874672572580352/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787829049.730482:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829049.730486:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787829049.730488:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829049.730492:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 2001 00000100:00080000:3.0:1787829049.730496:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829049.730502:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:3.0:1787829049.730513:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787829049.730519:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb9923170c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572580352:192.168.203.129@tcp:250:kworker.0 00000100:00100000:3.0:1787829049.730532:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb9923161c0 x1874672572580480/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787829049.730555:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829049.730559:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1787829049.730561:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829049.730564:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 2001 00000100:00080000:3.0:1787829049.730568:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829049.730571:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:3.0:1787829049.730576:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787829049.730590:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb992314f00 x1874672572580608/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1787829049.730609:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787829049.730649:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829049/real 1787829049] req@ffff8cb9923170c0 x1874672572580352/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829074 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829049.730668:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb9923170c0 x1874672572580352/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829074 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829049.730678:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829049.730683:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787829049.730687:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb9923170c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572580352:192.168.203.129@tcp:250:kworker.0 00000100:00100000:2.0:1787829049.730710:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb9923161c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572580480:192.168.203.129@tcp:38:kworker.0 00000800:00000100:2.0:1787829049.730735:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787829049.730748:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829049/real 1787829049] req@ffff8cb9923161c0 x1874672572580480/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829074 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829049.730760:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb9923161c0 x1874672572580480/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829074 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829049.730767:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829049.730770:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787829049.730780:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb9923161c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572580480:192.168.203.129@tcp:38:kworker.0 00000100:00100000:2.0:1787829049.730786:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb992314f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572580608:192.168.203.129@tcp:38:kworker.0 00000800:00000100:2.0:1787829049.730807:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787829049.730818:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829049/real 1787829049] req@ffff8cb992314f00 x1874672572580608/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829074 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829049.730847:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb992314f00 x1874672572580608/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829074 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829049.730855:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829049.730859:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787829049.730862:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb992314f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572580608:192.168.203.129@tcp:38:kworker.0 00010000:00010000:3.0:1787829050.562825:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787829051.586806:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787829052.610534:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787829053.634622:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829054.658618:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787829054.850362:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829054.850377:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787829054.850381:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829054.850389:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 2007 00000100:00080000:1.0:1787829054.850398:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829054.850403:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:1.0:1787829054.850438:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f38b40 x1874672572580736/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829054.850457:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829054.850460:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787829054.850474:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829054.850478:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 2007 00000100:00080000:1.0:1787829054.850482:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829054.850485:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:1.0:1787829054.850495:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787829054.850512:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f39e00 x1874672572580864/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787829054.850512:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f38b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572580736:192.168.203.129@tcp:250:kworker.0 00000100:00080000:1.0:1787829054.850532:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829054.850535:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1787829054.850538:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829054.850542:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 2007 00000100:00080000:1.0:1787829054.850546:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829054.850550:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:1.0:1787829054.850559:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787829054.850571:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f3b840 x1874672572580992/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1787829054.850595:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787829054.850645:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829054/real 1787829054] req@ffff8cb999f38b40 x1874672572580736/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829079 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829054.850665:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f38b40 x1874672572580736/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829079 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829054.850685:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829054.850691:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787829054.850695:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f38b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572580736:192.168.203.129@tcp:250:kworker.0 00000100:00100000:2.0:1787829054.850722:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f39e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572580864:192.168.203.129@tcp:38:kworker.0 00000800:00000100:2.0:1787829054.850746:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787829054.850760:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829054/real 1787829054] req@ffff8cb999f39e00 x1874672572580864/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829079 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829054.850773:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f39e00 x1874672572580864/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829079 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829054.850795:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829054.850799:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787829054.850803:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f39e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572580864:192.168.203.129@tcp:38:kworker.0 00000100:00100000:2.0:1787829054.850809:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f3b840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572580992:192.168.203.129@tcp:38:kworker.0 00000800:00000100:2.0:1787829054.850833:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787829054.850846:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829054/real 1787829054] req@ffff8cb999f3b840 x1874672572580992/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829079 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829054.850868:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f3b840 x1874672572580992/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829079 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829054.850876:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829054.850880:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787829054.850883:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f3b840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572580992:192.168.203.129@tcp:38:kworker.0 00010000:00010000:1.0:1787829055.682790:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787829056.706611:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829057.730403:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787829058.754370:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787829059.778364:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787829059.970353:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829059.970362:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787829059.970367:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829059.970376:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 2012 00000100:00080000:3.0:1787829059.970387:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829059.970394:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:3.0:1787829059.970428:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb992315a40 x1874672572581120/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787829059.970453:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829059.970456:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787829059.970459:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829059.970463:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 2012 00000100:00080000:3.0:1787829059.970466:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829059.970470:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:3.0:1787829059.970480:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787829059.970493:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb992315a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572581120:192.168.203.129@tcp:250:kworker.0 00000100:00100000:3.0:1787829059.970495:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb992316940 x1874672572581248/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787829059.970509:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829059.970513:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1787829059.970529:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829059.970534:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 2012 00000100:00080000:3.0:1787829059.970538:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829059.970543:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:3.0:1787829059.970549:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000800:00000100:2.0:1787829059.970559:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:3.0:1787829059.970563:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb992315e00 x1874672572581376/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787829059.970607:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829059/real 1787829059] req@ffff8cb992315a40 x1874672572581120/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829084 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829059.970628:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb992315a40 x1874672572581120/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829084 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829059.970640:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829059.970644:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787829059.970660:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb992315a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572581120:192.168.203.129@tcp:250:kworker.0 00000100:00100000:2.0:1787829059.970684:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb992316940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572581248:192.168.203.129@tcp:38:kworker.0 00000800:00000100:2.0:1787829059.970704:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787829059.970714:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829059/real 1787829059] req@ffff8cb992316940 x1874672572581248/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829084 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829059.970723:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb992316940 x1874672572581248/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829084 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829059.970729:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829059.970732:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787829059.970734:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb992316940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572581248:192.168.203.129@tcp:38:kworker.0 00000100:00100000:2.0:1787829059.970738:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb992315e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572581376:192.168.203.129@tcp:38:kworker.0 00000800:00000100:2.0:1787829059.970750:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787829059.970756:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829059/real 1787829059] req@ffff8cb992315e00 x1874672572581376/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829084 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829059.970763:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb992315e00 x1874672572581376/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829084 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829059.970768:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829059.970770:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787829059.970772:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb992315e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572581376:192.168.203.129@tcp:38:kworker.0 00010000:00010000:2.0:1787829060.802399:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787829061.826489:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787829062.850370:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829063.874475:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787829064.898389:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787829065.090408:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829065.090419:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787829065.090423:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829065.090434:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 2017 00000100:00080000:3.0:1787829065.090446:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829065.090453:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:3.0:1787829065.090499:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb992317480 x1874672572581504/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787829065.090533:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829065.090536:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787829065.090537:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829065.090540:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 2017 00000100:00080000:3.0:1787829065.090543:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829065.090545:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:3.0:1787829065.090554:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787829065.090563:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb992314b40 x1874672572581632/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787829065.090595:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829065.090598:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1787829065.090599:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829065.090603:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 2017 00000100:00080000:3.0:1787829065.090606:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829065.090609:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:3.0:1787829065.090613:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787829065.090627:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb992314780 x1874672572581760/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787829065.090629:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb992317480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572581504:192.168.203.129@tcp:250:kworker.0 00000800:00000100:2.0:1787829065.090716:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787829065.090786:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829065/real 1787829065] req@ffff8cb992317480 x1874672572581504/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829090 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829065.090815:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb992317480 x1874672572581504/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829090 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829065.090824:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829065.090828:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787829065.090833:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb992317480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572581504:192.168.203.129@tcp:250:kworker.0 00000100:00100000:2.0:1787829065.090841:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb992314b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572581632:192.168.203.129@tcp:38:kworker.0 00000800:00000100:2.0:1787829065.090872:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787829065.090901:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829065/real 1787829065] req@ffff8cb992314b40 x1874672572581632/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829090 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829065.090916:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb992314b40 x1874672572581632/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829090 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829065.090921:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829065.090924:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787829065.090928:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb992314b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572581632:192.168.203.129@tcp:38:kworker.0 00000100:00100000:2.0:1787829065.090985:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb992314780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572581760:192.168.203.129@tcp:38:kworker.0 00000800:00000100:2.0:1787829065.091018:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787829065.091028:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829065/real 1787829065] req@ffff8cb992314780 x1874672572581760/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829090 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829065.091061:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb992314780 x1874672572581760/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829090 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829065.091074:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829065.091077:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787829065.091079:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb992314780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572581760:192.168.203.129@tcp:38:kworker.0 00010000:00010000:3.0:1787829065.922365:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829066.946431:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787829067.970374:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787829068.994389:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829070.018408:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787829070.210345:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829070.210353:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787829070.210368:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829070.210376:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 2022 00000100:00080000:3.0:1787829070.210386:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829070.210391:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:3.0:1787829070.210435:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb992316580 x1874672572581888/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787829070.210461:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829070.210464:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787829070.210465:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829070.210468:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 2022 00000100:00080000:3.0:1787829070.210485:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829070.210489:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:3.0:1787829070.210498:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787829070.210513:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb992316580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572581888:192.168.203.129@tcp:250:kworker.0 00000100:00100000:3.0:1787829070.210513:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb992314780 x1874672572582016/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787829070.210534:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829070.210537:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1787829070.210538:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829070.210542:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 2022 00000100:00080000:3.0:1787829070.210545:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829070.210547:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:3.0:1787829070.210551:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787829070.210561:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb992314b40 x1874672572582144/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1787829070.210576:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787829070.210629:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829070/real 1787829070] req@ffff8cb992316580 x1874672572581888/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829095 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829070.210650:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb992316580 x1874672572581888/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829095 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829070.210662:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829070.210666:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787829070.210670:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb992316580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572581888:192.168.203.129@tcp:250:kworker.0 00000100:00100000:2.0:1787829070.210695:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb992314780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572582016:192.168.203.129@tcp:38:kworker.0 00000800:00000100:2.0:1787829070.210730:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787829070.210745:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829070/real 1787829070] req@ffff8cb992314780 x1874672572582016/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829095 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829070.210759:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb992314780 x1874672572582016/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829095 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829070.210787:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829070.210791:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787829070.210795:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb992314780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572582016:192.168.203.129@tcp:38:kworker.0 00000100:00100000:2.0:1787829070.210803:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb992314b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572582144:192.168.203.129@tcp:38:kworker.0 00000800:00000100:2.0:1787829070.210826:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787829070.210836:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829070/real 1787829070] req@ffff8cb992314b40 x1874672572582144/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829095 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829070.210844:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb992314b40 x1874672572582144/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829095 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829070.210850:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829070.210853:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787829070.210855:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb992314b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572582144:192.168.203.129@tcp:38:kworker.0 00010000:00010000:1.0:1787829071.042469:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787829072.066434:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829073.090458:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787829074.114468:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787829075.138422:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787829075.330381:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829075.330396:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787829075.330443:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829075.330454:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 2027 00000100:00080000:1.0:1787829075.330467:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829075.330474:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:1.0:1787829075.330514:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f39a40 x1874672572582272/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829075.330543:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829075.330546:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787829075.330548:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829075.330551:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 2027 00000100:00080000:1.0:1787829075.330553:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829075.330556:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:1.0:1787829075.330566:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787829075.330577:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f3b840 x1874672572582400/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829075.330585:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829075.330588:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1787829075.330590:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829075.330593:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 2027 00000100:00080000:1.0:1787829075.330595:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829075.330598:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:1.0:1787829075.330601:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787829075.330607:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f39a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572582272:192.168.203.129@tcp:250:kworker.0 00000100:00100000:1.0:1787829075.330615:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f39e00 x1874672572582528/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1787829075.330690:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787829075.330735:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829075/real 1787829075] req@ffff8cb999f39a40 x1874672572582272/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829100 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829075.330751:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f39a40 x1874672572582272/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829100 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829075.330760:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829075.330763:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787829075.330773:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f39a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572582272:192.168.203.129@tcp:250:kworker.0 00000100:00100000:2.0:1787829075.330779:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f3b840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572582400:192.168.203.129@tcp:38:kworker.0 00000800:00000100:2.0:1787829075.330798:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787829075.330807:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829075/real 1787829075] req@ffff8cb999f3b840 x1874672572582400/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829100 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829075.330816:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f3b840 x1874672572582400/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829100 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829075.330821:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829075.330824:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787829075.330826:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f3b840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572582400:192.168.203.129@tcp:38:kworker.0 00000100:00100000:2.0:1787829075.330849:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f39e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572582528:192.168.203.129@tcp:38:kworker.0 00000800:00000100:2.0:1787829075.330861:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787829075.330867:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829075/real 1787829075] req@ffff8cb999f39e00 x1874672572582528/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829100 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829075.330875:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f39e00 x1874672572582528/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829100 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829075.330880:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829075.330882:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787829075.330885:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f39e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572582528:192.168.203.129@tcp:38:kworker.0 00010000:00010000:2.0:1787829076.162421:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787829077.186524:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787829078.210367:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829079.234423:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787829080.258415:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787829080.450417:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829080.450428:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787829080.450432:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829080.450458:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 2032 00000100:00080000:1.0:1787829080.450468:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829080.450473:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:1.0:1787829080.450528:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f38b40 x1874672572582656/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829080.450562:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829080.450583:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787829080.450585:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829080.450588:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 2032 00000100:00080000:1.0:1787829080.450591:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829080.450594:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:1.0:1787829080.450604:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787829080.450615:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f3b480 x1874672572582784/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829080.450624:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829080.450627:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1787829080.450629:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829080.450648:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 2032 00000100:00100000:2.0:1787829080.450655:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f38b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572582656:192.168.203.129@tcp:250:kworker.0 00000100:00080000:1.0:1787829080.450667:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829080.450672:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:1.0:1787829080.450677:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787829080.450689:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f3b0c0 x1874672572582912/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1787829080.450719:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787829080.450773:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829080/real 1787829080] req@ffff8cb999f38b40 x1874672572582656/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829105 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829080.450789:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f38b40 x1874672572582656/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829105 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829080.450799:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829080.450802:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787829080.450806:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f38b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572582656:192.168.203.129@tcp:250:kworker.0 00000100:00100000:2.0:1787829080.450812:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f3b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572582784:192.168.203.129@tcp:38:kworker.0 00000800:00000100:2.0:1787829080.450830:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787829080.450839:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829080/real 1787829080] req@ffff8cb999f3b480 x1874672572582784/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829105 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829080.450848:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f3b480 x1874672572582784/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829105 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829080.450855:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829080.450858:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787829080.450860:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f3b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572582784:192.168.203.129@tcp:38:kworker.0 00000100:00100000:2.0:1787829080.450880:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f3b0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572582912:192.168.203.129@tcp:38:kworker.0 00000800:00000100:2.0:1787829080.450893:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787829080.450900:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829080/real 1787829080] req@ffff8cb999f3b0c0 x1874672572582912/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829105 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829080.450922:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f3b0c0 x1874672572582912/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829105 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829080.450928:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829080.450931:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787829080.450933:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f3b0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572582912:192.168.203.129@tcp:38:kworker.0 00010000:00010000:3.0:1787829081.282379:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829082.306421:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787829083.330364:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787829084.354386:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829085.378396:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787829085.570356:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829085.570364:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787829085.570367:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829085.570376:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 2037 00000100:00080000:1.0:1787829085.570385:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829085.570390:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:1.0:1787829085.570421:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f38f00 x1874672572583040/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829085.570446:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829085.570450:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787829085.570452:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829085.570455:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 2037 00000100:00080000:1.0:1787829085.570459:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829085.570463:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:1.0:1787829085.570472:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787829085.570484:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f392c0 x1874672572583168/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829085.570494:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1787829085.570495:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f38f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572583040:192.168.203.129@tcp:250:kworker.0 00000100:00080000:1.0:1787829085.570509:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1787829085.570511:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829085.570515:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 2037 00000100:00080000:1.0:1787829085.570519:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829085.570522:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:1.0:1787829085.570527:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787829085.570536:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f3a580 x1874672572583296/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1787829085.570552:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787829085.570590:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829085/real 1787829085] req@ffff8cb999f38f00 x1874672572583040/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829110 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829085.570603:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f38f00 x1874672572583040/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829110 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829085.570611:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829085.570614:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787829085.570617:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f38f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572583040:192.168.203.129@tcp:250:kworker.0 00000100:00100000:2.0:1787829085.570638:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f392c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572583168:192.168.203.129@tcp:38:kworker.0 00000800:00000100:2.0:1787829085.570661:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787829085.570675:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829085/real 1787829085] req@ffff8cb999f392c0 x1874672572583168/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829110 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829085.570686:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f392c0 x1874672572583168/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829110 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829085.570701:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829085.570705:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787829085.570708:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f392c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572583168:192.168.203.129@tcp:38:kworker.0 00000100:00100000:2.0:1787829085.570713:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f3a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572583296:192.168.203.129@tcp:38:kworker.0 00000800:00000100:2.0:1787829085.570735:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787829085.570747:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829085/real 1787829085] req@ffff8cb999f3a580 x1874672572583296/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829110 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829085.570760:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f3a580 x1874672572583296/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829110 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829085.570769:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829085.570772:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787829085.570776:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f3a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572583296:192.168.203.129@tcp:38:kworker.0 00010000:00010000:1.0:1787829086.402388:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787829087.426366:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829088.450453:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787829089.474396:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787829090.498409:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787829090.690401:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829090.690418:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787829090.690423:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829090.690436:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 2042 00000100:00080000:1.0:1787829090.690450:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829090.690459:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:1.0:1787829090.690512:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f383c0 x1874672572583424/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829090.690547:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829090.690550:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787829090.690552:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829090.690555:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 2042 00000100:00080000:1.0:1787829090.690558:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829090.690561:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:1.0:1787829090.690594:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787829090.690599:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f383c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572583424:192.168.203.129@tcp:250:kworker.0 00000100:00100000:1.0:1787829090.690614:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f3a580 x1874672572583552/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829090.690628:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829090.690632:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1787829090.690634:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829090.690638:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 2042 00000100:00080000:1.0:1787829090.690643:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829090.690648:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:1.0:1787829090.690654:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787829090.690669:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f392c0 x1874672572583680/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1787829090.690674:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787829090.690730:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829090/real 1787829090] req@ffff8cb999f383c0 x1874672572583424/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829115 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829090.690752:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f383c0 x1874672572583424/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829115 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829090.690762:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829090.690766:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787829090.690776:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f383c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572583424:192.168.203.129@tcp:250:kworker.0 00000100:00100000:2.0:1787829090.690801:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f3a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572583552:192.168.203.129@tcp:38:kworker.0 00000800:00000100:2.0:1787829090.690823:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787829090.690838:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829090/real 1787829090] req@ffff8cb999f3a580 x1874672572583552/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829115 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829090.690852:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f3a580 x1874672572583552/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829115 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829090.690862:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829090.690866:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787829090.690869:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f3a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572583552:192.168.203.129@tcp:38:kworker.0 00000100:00100000:2.0:1787829090.690876:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f392c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572583680:192.168.203.129@tcp:38:kworker.0 00000800:00000100:2.0:1787829090.690910:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787829090.690924:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829090/real 1787829090] req@ffff8cb999f392c0 x1874672572583680/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829115 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829090.690936:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f392c0 x1874672572583680/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829115 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829090.690944:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829090.690948:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787829090.690951:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f392c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572583680:192.168.203.129@tcp:38:kworker.0 00010000:00010000:2.0:1787829091.522473:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787829092.546489:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787829093.570405:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829094.594424:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787829095.618444:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787829095.810371:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829095.810379:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787829095.810384:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829095.810399:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 2047 00000100:00080000:1.0:1787829095.810411:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829095.810418:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:1.0:1787829095.810463:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f38f00 x1874672572583808/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829095.810485:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829095.810488:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787829095.810502:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829095.810505:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 2047 00000100:00080000:1.0:1787829095.810508:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829095.810512:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:1.0:1787829095.810520:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787829095.810529:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f3b0c0 x1874672572583936/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829095.810536:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829095.810539:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1787829095.810540:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829095.810542:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 2047 00000100:00080000:1.0:1787829095.810545:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829095.810547:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:1.0:1787829095.810550:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787829095.810557:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f3b480 x1874672572584064/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787829095.810605:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f38f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572583808:192.168.203.129@tcp:250:kworker.0 00000800:00000100:2.0:1787829095.810696:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787829095.810769:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829095/real 1787829095] req@ffff8cb999f38f00 x1874672572583808/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829120 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829095.810791:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f38f00 x1874672572583808/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829120 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829095.810804:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829095.810809:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787829095.810813:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f38f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572583808:192.168.203.129@tcp:250:kworker.0 00000100:00100000:2.0:1787829095.810821:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f3b0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572583936:192.168.203.129@tcp:38:kworker.0 00000800:00000100:2.0:1787829095.810847:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787829095.810860:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829095/real 1787829095] req@ffff8cb999f3b0c0 x1874672572583936/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829120 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829095.810873:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f3b0c0 x1874672572583936/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829120 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829095.810882:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829095.810885:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787829095.810889:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f3b0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572583936:192.168.203.129@tcp:38:kworker.0 00000100:00100000:2.0:1787829095.810895:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f3b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572584064:192.168.203.129@tcp:38:kworker.0 00000800:00000100:2.0:1787829095.810911:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787829095.810919:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829095/real 1787829095] req@ffff8cb999f3b480 x1874672572584064/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829120 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829095.810962:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f3b480 x1874672572584064/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829120 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829095.810968:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829095.810970:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787829095.810972:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f3b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572584064:192.168.203.129@tcp:38:kworker.0 00010000:00010000:3.0:1787829096.642388:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829097.666423:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787829098.690434:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787829099.714371:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829100.738502:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787829100.930326:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829100.930337:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787829100.930342:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829100.930352:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 2053 00000100:00080000:1.0:1787829100.930363:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829100.930370:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:1.0:1787829100.930410:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f38b40 x1874672572584192/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829100.930461:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829100.930466:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787829100.930468:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829100.930471:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 2053 00000100:00080000:1.0:1787829100.930475:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829100.930479:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:1.0:1787829100.930497:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787829100.930509:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f39e00 x1874672572584320/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829100.930519:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829100.930522:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1787829100.930523:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829100.930528:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 2053 00000100:00080000:1.0:1787829100.930531:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829100.930534:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:1.0:1787829100.930539:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787829100.930546:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f38b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572584192:192.168.203.129@tcp:250:kworker.0 00000100:00100000:1.0:1787829100.930549:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f3b840 x1874672572584448/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1787829100.930633:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787829100.930685:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829100/real 1787829100] req@ffff8cb999f38b40 x1874672572584192/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829125 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829100.930702:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f38b40 x1874672572584192/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829125 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829100.930711:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829100.930715:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787829100.930718:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f38b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572584192:192.168.203.129@tcp:250:kworker.0 00000100:00100000:2.0:1787829100.930743:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f39e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572584320:192.168.203.129@tcp:38:kworker.0 00000800:00000100:2.0:1787829100.930768:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787829100.930790:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829100/real 1787829100] req@ffff8cb999f39e00 x1874672572584320/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829125 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829100.930800:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f39e00 x1874672572584320/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829125 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829100.930820:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829100.930824:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787829100.930828:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f39e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572584320:192.168.203.129@tcp:38:kworker.0 00000100:00100000:2.0:1787829100.930834:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f3b840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572584448:192.168.203.129@tcp:38:kworker.0 00000800:00000100:2.0:1787829100.930851:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787829100.930876:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829100/real 1787829100] req@ffff8cb999f3b840 x1874672572584448/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829125 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829100.930893:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f3b840 x1874672572584448/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829125 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829100.930900:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829100.930903:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787829100.930920:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f3b840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572584448:192.168.203.129@tcp:38:kworker.0 00010000:00010000:2.0:1787829101.762435:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787829102.786410:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829103.810464:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829104.834399:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787829105.858428:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787829106.050360:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829106.050373:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787829106.050378:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829106.050390:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 2058 00000100:00080000:3.0:1787829106.050402:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829106.050410:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:3.0:1787829106.050457:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb992317480 x1874672572584576/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787829106.050487:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829106.050491:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787829106.050492:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829106.050495:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 2058 00000100:00080000:3.0:1787829106.050498:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829106.050501:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:3.0:1787829106.050511:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787829106.050520:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb992315e00 x1874672572584704/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787829106.050527:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829106.050530:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1787829106.050531:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829106.050534:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 2058 00000100:00080000:3.0:1787829106.050537:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829106.050540:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:3.0:1787829106.050542:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787829106.050561:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb992317480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572584576:192.168.203.129@tcp:250:kworker.0 00000100:00100000:3.0:1787829106.050575:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb992316940 x1874672572584832/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1787829106.050670:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787829106.050718:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829106/real 1787829106] req@ffff8cb992317480 x1874672572584576/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829131 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829106.050735:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb992317480 x1874672572584576/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829131 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829106.050756:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829106.050760:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787829106.050763:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb992317480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572584576:192.168.203.129@tcp:250:kworker.0 00000100:00100000:2.0:1787829106.050769:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb992315e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572584704:192.168.203.129@tcp:38:kworker.0 00000800:00000100:2.0:1787829106.050789:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787829106.050798:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829106/real 1787829106] req@ffff8cb992315e00 x1874672572584704/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829131 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829106.050806:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb992315e00 x1874672572584704/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829131 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829106.050812:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829106.050814:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787829106.050816:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb992315e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572584704:192.168.203.129@tcp:38:kworker.0 00000100:00100000:2.0:1787829106.050843:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb992316940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572584832:192.168.203.129@tcp:38:kworker.0 00000800:00000100:2.0:1787829106.050854:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787829106.050860:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829106/real 1787829106] req@ffff8cb992316940 x1874672572584832/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829131 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829106.050867:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb992316940 x1874672572584832/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829131 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829106.050891:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829106.050893:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787829106.050896:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb992316940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572584832:192.168.203.129@tcp:38:kworker.0 00010000:00010000:2.0:1787829106.882434:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829107.906527:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787829108.930366:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829109.954506:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829110.978453:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787829111.170415:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829111.170429:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787829111.170434:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829111.170447:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 2063 00000100:00080000:1.0:1787829111.170486:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829111.170495:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:1.0:1787829111.170541:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f39a40 x1874672572584960/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829111.170580:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829111.170584:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787829111.170586:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829111.170591:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 2063 00000100:00080000:1.0:1787829111.170595:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829111.170600:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:1.0:1787829111.170614:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787829111.170630:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f39a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572584960:192.168.203.129@tcp:250:kworker.0 00000100:00100000:1.0:1787829111.170633:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f3b840 x1874672572585088/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829111.170648:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829111.170652:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1787829111.170655:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829111.170659:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 2063 00000100:00080000:1.0:1787829111.170664:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829111.170668:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:1.0:1787829111.170676:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787829111.170689:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f39e00 x1874672572585216/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787829111.170782:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f3b840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572585088:192.168.203.129@tcp:38:kworker.0 00000100:00100000:2.0:1787829111.170824:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f39e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572585216:192.168.203.129@tcp:38:kworker.0 00010000:00010000:0.0:1787829112.002403:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829113.026462:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829114.051126:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:3.0:1787829114.242482:0:2383:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.129:988 00000400:00000100:3.0:1787829114.242524:0:2383:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.129@tcp at host 192.168.203.129 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:3.0:1787829114.242540:0:2383:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.129@tcp 00000800:00000100:3.0:1787829114.242629:0:2383:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.129@tcp 00000800:00000100:3.0:1787829114.242642:0:2383:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.129@tcp 00000100:00100000:2.0:1787829114.242668:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829111/real 1787829114] req@ffff8cb999f3b840 x1874672572585088/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829136 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829114.242697:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f3b840 x1874672572585088/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829136 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829114.242710:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829114.242716:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787829114.242721:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f3b840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572585088:192.168.203.129@tcp:38:kworker.0 00000100:00100000:2.0:1787829114.242729:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829111/real 1787829114] req@ffff8cb999f39e00 x1874672572585216/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829136 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829114.242742:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f39e00 x1874672572585216/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829136 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829114.242751:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829114.242755:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787829114.242775:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f39e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572585216:192.168.203.129@tcp:38:kworker.0 00000100:00100000:2.0:1787829114.242782:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829111/real 1787829114] req@ffff8cb999f39a40 x1874672572584960/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829136 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829114.242798:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f39a40 x1874672572584960/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829136 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829114.242806:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829114.242810:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787829114.242828:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f39a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572584960:192.168.203.129@tcp:250:kworker.0 00010000:00010000:0.0:1787829115.074953:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829116.098542:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787829116.290401:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829116.290413:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787829116.290417:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829116.290427:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 2068 00000100:00080000:2.0:1787829116.290438:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829116.290444:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:2.0:1787829116.290487:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8c000 x1874672572585344/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829116.290513:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829116.290516:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787829116.290517:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829116.290520:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 2068 00000100:00080000:2.0:1787829116.290523:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829116.290526:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:2.0:1787829116.290553:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787829116.290560:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572585344:192.168.203.129@tcp:250:kworker.0 00000100:00100000:2.0:1787829116.290566:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8f480 x1874672572585472/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829116.290584:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829116.290586:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1787829116.290588:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829116.290591:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 2068 00000100:00080000:2.0:1787829116.290593:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829116.290596:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:2.0:1787829116.290599:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787829116.290608:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8e940 x1874672572585600/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1787829116.290656:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787829116.290709:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829116/real 1787829116] req@ffff8cb998a8c000 x1874672572585344/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829141 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829116.290727:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8c000 x1874672572585344/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829141 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829116.290736:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829116.290740:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787829116.290744:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572585344:192.168.203.129@tcp:250:kworker.0 00000100:00100000:1.0:1787829116.290770:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572585472:192.168.203.129@tcp:38:kworker.0 00000800:00000100:1.0:1787829116.290795:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787829116.290827:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829116/real 1787829116] req@ffff8cb998a8f480 x1874672572585472/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829141 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829116.290841:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8f480 x1874672572585472/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829141 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829116.290850:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829116.290853:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787829116.290856:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572585472:192.168.203.129@tcp:38:kworker.0 00000100:00100000:1.0:1787829116.290863:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8e940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572585600:192.168.203.129@tcp:38:kworker.0 00000800:00000100:1.0:1787829116.290884:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787829116.290897:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829116/real 1787829116] req@ffff8cb998a8e940 x1874672572585600/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829141 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829116.290922:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8e940 x1874672572585600/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829141 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829116.290930:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829116.290933:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787829116.290937:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8e940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572585600:192.168.203.129@tcp:38:kworker.0 00010000:00010000:2.0:1787829117.122725:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787829118.146752:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787829119.170655:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829120.194535:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787829121.218576:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787829121.410383:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829121.410391:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787829121.410395:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829121.410403:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 2073 00000100:00080000:1.0:1787829121.410412:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829121.410418:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:1.0:1787829121.410450:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f38b40 x1874672572585728/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829121.410473:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829121.410476:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787829121.410478:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829121.410481:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 2073 00000100:00080000:1.0:1787829121.410485:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829121.410490:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:1.0:1787829121.410500:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787829121.410507:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f38b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572585728:192.168.203.129@tcp:250:kworker.0 00000100:00100000:1.0:1787829121.410516:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f3b480 x1874672572585856/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829121.410542:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829121.410545:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1787829121.410547:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829121.410551:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 2073 00000100:00080000:1.0:1787829121.410558:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829121.410563:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000800:00000100:2.0:1787829121.410568:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000008:00100000:1.0:1787829121.410573:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787829121.410591:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f3b0c0 x1874672572585984/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787829121.410609:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829121/real 1787829121] req@ffff8cb999f38b40 x1874672572585728/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829146 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829121.410632:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f38b40 x1874672572585728/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829146 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829121.410645:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829121.410649:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787829121.410654:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f38b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572585728:192.168.203.129@tcp:250:kworker.0 00000100:00100000:2.0:1787829121.410682:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f3b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572585856:192.168.203.129@tcp:38:kworker.0 00000800:00000100:2.0:1787829121.410701:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787829121.410711:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829121/real 1787829121] req@ffff8cb999f3b480 x1874672572585856/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829146 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829121.410720:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f3b480 x1874672572585856/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829146 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829121.410726:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829121.410728:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787829121.410730:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f3b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572585856:192.168.203.129@tcp:38:kworker.0 00000100:00100000:2.0:1787829121.410734:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f3b0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572585984:192.168.203.129@tcp:38:kworker.0 00000800:00000100:2.0:1787829121.410746:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787829121.410751:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829121/real 1787829121] req@ffff8cb999f3b0c0 x1874672572585984/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829146 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829121.410759:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f3b0c0 x1874672572585984/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829146 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829121.410779:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829121.410783:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787829121.410786:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f3b0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572585984:192.168.203.129@tcp:38:kworker.0 00010000:00010000:2.0:1787829122.242632:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829123.266687:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787829124.290388:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787829125.314414:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829126.338362:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787829126.530377:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829126.530385:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787829126.530388:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829126.530396:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 2078 00000100:00080000:3.0:1787829126.530405:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829126.530410:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:3.0:1787829126.530441:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb992315a40 x1874672572586112/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787829126.530461:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829126.530464:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787829126.530465:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829126.530468:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 2078 00000100:00080000:3.0:1787829126.530470:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829126.530472:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:3.0:1787829126.530479:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787829126.530488:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb992314f00 x1874672572586240/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787829126.530508:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb992315a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572586112:192.168.203.129@tcp:250:kworker.0 00000100:00080000:3.0:1787829126.530511:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829126.530514:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1787829126.530515:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829126.530523:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 2078 00000100:00080000:3.0:1787829126.530533:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829126.530536:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:3.0:1787829126.530543:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787829126.530553:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb9923161c0 x1874672572586368/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1787829126.530573:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787829126.530658:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829126/real 1787829126] req@ffff8cb992315a40 x1874672572586112/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829151 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829126.530672:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb992315a40 x1874672572586112/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829151 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829126.530680:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829126.530684:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787829126.530687:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb992315a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572586112:192.168.203.129@tcp:250:kworker.0 00000100:00100000:2.0:1787829126.530706:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb992314f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572586240:192.168.203.129@tcp:38:kworker.0 00000800:00000100:2.0:1787829126.530723:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787829126.530747:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829126/real 1787829126] req@ffff8cb992314f00 x1874672572586240/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829151 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829126.530757:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb992314f00 x1874672572586240/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829151 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829126.530763:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829126.530765:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787829126.530767:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb992314f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572586240:192.168.203.129@tcp:38:kworker.0 00000100:00100000:2.0:1787829126.530771:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb9923161c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572586368:192.168.203.129@tcp:38:kworker.0 00000800:00000100:2.0:1787829126.530788:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787829126.530800:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829126/real 1787829126] req@ffff8cb9923161c0 x1874672572586368/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829151 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829126.530827:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb9923161c0 x1874672572586368/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829151 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829126.530836:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829126.530839:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787829126.530841:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb9923161c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572586368:192.168.203.129@tcp:38:kworker.0 00010000:00010000:1.0:1787829127.362410:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787829128.386383:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829129.410401:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787829130.434397:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787829131.458585:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787829131.650357:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829131.650366:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787829131.650369:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829131.650377:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 2083 00000100:00080000:3.0:1787829131.650386:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829131.650394:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:3.0:1787829131.650433:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb9923170c0 x1874672572586496/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787829131.650457:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829131.650460:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787829131.650461:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829131.650464:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 2083 00000100:00080000:3.0:1787829131.650467:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829131.650470:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:3.0:1787829131.650493:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787829131.650497:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb9923170c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572586496:192.168.203.129@tcp:250:kworker.0 00000100:00100000:3.0:1787829131.650511:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb9923161c0 x1874672572586624/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787829131.650525:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829131.650529:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1787829131.650532:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829131.650536:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 2083 00000100:00080000:3.0:1787829131.650549:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829131.650553:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:3.0:1787829131.650557:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000800:00000100:2.0:1787829131.650563:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:3.0:1787829131.650568:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb992314f00 x1874672572586752/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787829131.650601:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829131/real 1787829131] req@ffff8cb9923170c0 x1874672572586496/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829156 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829131.650615:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb9923170c0 x1874672572586496/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829156 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829131.650623:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829131.650625:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787829131.650628:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb9923170c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572586496:192.168.203.129@tcp:250:kworker.0 00000100:00100000:2.0:1787829131.650648:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb9923161c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572586624:192.168.203.129@tcp:38:kworker.0 00000800:00000100:2.0:1787829131.650664:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787829131.650672:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829131/real 1787829131] req@ffff8cb9923161c0 x1874672572586624/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829156 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829131.650681:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb9923161c0 x1874672572586624/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829156 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829131.650711:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829131.650714:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787829131.650716:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb9923161c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572586624:192.168.203.129@tcp:38:kworker.0 00000100:00100000:2.0:1787829131.650721:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb992314f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572586752:192.168.203.129@tcp:38:kworker.0 00000800:00000100:2.0:1787829131.650739:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787829131.650751:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829131/real 1787829131] req@ffff8cb992314f00 x1874672572586752/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829156 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829131.650764:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb992314f00 x1874672572586752/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829156 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829131.650772:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829131.650776:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787829131.650779:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb992314f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572586752:192.168.203.129@tcp:38:kworker.0 00010000:00010000:2.0:1787829132.482412:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787829133.506431:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787829134.530478:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787829135.554454:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787829136.578452:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787829136.770373:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829136.770380:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787829136.770384:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829136.770392:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 2088 00000100:00080000:1.0:1787829136.770402:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829136.770409:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:1.0:1787829136.770439:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f38f00 x1874672572586880/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829136.770466:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829136.770470:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787829136.770486:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829136.770490:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 2088 00000100:00080000:1.0:1787829136.770494:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829136.770497:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:3.0:1787829136.770502:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f38f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572586880:192.168.203.129@tcp:250:kworker.0 00000008:00100000:1.0:1787829136.770507:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787829136.770521:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f392c0 x1874672572587008/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829136.770530:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829136.770533:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1787829136.770534:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829136.770537:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 2088 00000100:00080000:1.0:1787829136.770540:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829136.770543:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:1.0:1787829136.770547:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787829136.770555:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f3a580 x1874672572587136/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1787829136.770565:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:3.0:1787829136.770614:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829136/real 1787829136] req@ffff8cb999f38f00 x1874672572586880/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829161 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787829136.770634:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f38f00 x1874672572586880/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829161 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787829136.770645:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829136.770648:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:3.0:1787829136.770652:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f38f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572586880:192.168.203.129@tcp:250:kworker.0 00000100:00100000:3.0:1787829136.770689:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f392c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572587008:192.168.203.129@tcp:38:kworker.0 00000800:00000100:3.0:1787829136.770714:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:3.0:1787829136.770724:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829136/real 1787829136] req@ffff8cb999f392c0 x1874672572587008/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829161 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787829136.770732:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f392c0 x1874672572587008/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829161 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787829136.770737:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829136.770740:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:3.0:1787829136.770742:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f392c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572587008:192.168.203.129@tcp:38:kworker.0 00000100:00100000:3.0:1787829136.770746:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f3a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572587136:192.168.203.129@tcp:38:kworker.0 00000800:00000100:3.0:1787829136.770756:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:3.0:1787829136.770762:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829136/real 1787829136] req@ffff8cb999f3a580 x1874672572587136/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829161 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787829136.770769:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f3a580 x1874672572587136/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829161 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787829136.770774:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829136.770776:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:3.0:1787829136.770777:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f3a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572587136:192.168.203.129@tcp:38:kworker.0 00010000:00010000:0.0:1787829137.602455:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787829138.626564:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787829139.650487:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787829140.674451:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787829141.698452:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787829141.890354:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829141.890363:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787829141.890366:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829141.890374:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 2094 00000100:00080000:2.0:1787829141.890383:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829141.890388:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:2.0:1787829141.890430:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8f840 x1874672572587264/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829141.890453:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829141.890467:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787829141.890469:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829141.890472:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 2094 00000100:00080000:2.0:1787829141.890475:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829141.890479:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:2.0:1787829141.890487:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787829141.890498:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8ed00 x1874672572587392/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829141.890506:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829141.890508:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1787829141.890509:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829141.890513:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 2094 00000100:00080000:2.0:1787829141.890515:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829141.890517:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:2.0:1787829141.890520:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787829141.890531:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8e940 x1874672572587520/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1787829141.890534:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8f840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572587264:192.168.203.129@tcp:250:kworker.0 00000800:00000100:3.0:1787829141.890638:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:3.0:1787829141.890723:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829141/real 1787829141] req@ffff8cb998a8f840 x1874672572587264/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829166 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787829141.890788:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8f840 x1874672572587264/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829166 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787829141.890819:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829141.890824:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:3.0:1787829141.890829:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8f840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572587264:192.168.203.129@tcp:250:kworker.0 00000100:00100000:3.0:1787829141.890839:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8ed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572587392:192.168.203.129@tcp:38:kworker.0 00000800:00000100:3.0:1787829141.890864:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:3.0:1787829141.890875:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829141/real 1787829141] req@ffff8cb998a8ed00 x1874672572587392/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829166 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787829141.890884:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8ed00 x1874672572587392/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829166 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787829141.890890:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829141.890893:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:3.0:1787829141.890895:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8ed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572587392:192.168.203.129@tcp:38:kworker.0 00000100:00100000:3.0:1787829141.890920:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8e940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572587520:192.168.203.129@tcp:38:kworker.0 00000800:00000100:3.0:1787829141.890933:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:3.0:1787829141.890940:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829141/real 1787829141] req@ffff8cb998a8e940 x1874672572587520/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829166 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787829141.890948:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8e940 x1874672572587520/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829166 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787829141.890980:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829141.890982:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:3.0:1787829141.890985:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8e940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572587520:192.168.203.129@tcp:38:kworker.0 00010000:00010000:0.0:1787829142.722507:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787829143.746438:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787829144.770429:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787829145.794448:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787829146.818434:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787829147.010391:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829147.010401:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787829147.010406:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829147.010416:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 2099 00000100:00080000:3.0:1787829147.010428:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829147.010434:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:3.0:1787829147.010477:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb992315a40 x1874672572587648/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787829147.010504:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829147.010508:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787829147.010509:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829147.010512:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 2099 00000100:00080000:3.0:1787829147.010515:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829147.010518:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:3.0:1787829147.010528:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787829147.010537:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb992316940 x1874672572587776/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787829147.010542:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb992315a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572587648:192.168.203.129@tcp:250:kworker.0 00000100:00080000:3.0:1787829147.010555:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829147.010558:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1787829147.010577:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829147.010582:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 2099 00000100:00080000:3.0:1787829147.010598:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829147.010601:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:3.0:1787829147.010606:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787829147.010617:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb992315e00 x1874672572587904/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1787829147.010631:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787829147.010685:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829147/real 1787829147] req@ffff8cb992315a40 x1874672572587648/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829172 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829147.010700:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb992315a40 x1874672572587648/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829172 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829147.010721:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829147.010725:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787829147.010728:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb992315a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572587648:192.168.203.129@tcp:250:kworker.0 00000100:00100000:2.0:1787829147.010753:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb992316940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572587776:192.168.203.129@tcp:38:kworker.0 00000800:00000100:2.0:1787829147.010771:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787829147.010782:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829147/real 1787829147] req@ffff8cb992316940 x1874672572587776/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829172 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829147.010790:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb992316940 x1874672572587776/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829172 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829147.010796:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829147.010798:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787829147.010800:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb992316940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572587776:192.168.203.129@tcp:38:kworker.0 00000100:00100000:2.0:1787829147.010804:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb992315e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572587904:192.168.203.129@tcp:38:kworker.0 00000800:00000100:2.0:1787829147.010815:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787829147.010821:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829147/real 1787829147] req@ffff8cb992315e00 x1874672572587904/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829172 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829147.010828:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb992315e00 x1874672572587904/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829172 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829147.010834:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829147.010835:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787829147.010837:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb992315e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572587904:192.168.203.129@tcp:38:kworker.0 00010000:00010000:0.0:1787829147.842420:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787829148.866451:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787829149.890493:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787829150.914388:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787829151.938416:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787829152.130399:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829152.130409:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787829152.130414:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829152.130424:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 2104 00000100:00080000:1.0:1787829152.130437:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829152.130443:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:1.0:1787829152.130486:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f383c0 x1874672572588032/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829152.130540:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829152.130543:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787829152.130545:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829152.130548:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 2104 00000100:00080000:1.0:1787829152.130567:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829152.130573:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:1.0:1787829152.130585:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787829152.130597:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f383c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572588032:192.168.203.129@tcp:250:kworker.0 00000100:00100000:1.0:1787829152.130607:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f3a580 x1874672572588160/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829152.130623:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829152.130628:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1787829152.130630:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829152.130637:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 2104 00000100:00080000:1.0:1787829152.130642:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829152.130649:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:1.0:1787829152.130657:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787829152.130670:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f392c0 x1874672572588288/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1787829152.130675:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787829152.130754:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829152/real 1787829152] req@ffff8cb999f383c0 x1874672572588032/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829177 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829152.130775:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f383c0 x1874672572588032/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829177 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829152.130784:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829152.130787:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787829152.130791:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f383c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572588032:192.168.203.129@tcp:250:kworker.0 00000100:00100000:2.0:1787829152.130812:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f3a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572588160:192.168.203.129@tcp:38:kworker.0 00000800:00000100:2.0:1787829152.130831:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787829152.130840:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829152/real 1787829152] req@ffff8cb999f3a580 x1874672572588160/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829177 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829152.130848:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f3a580 x1874672572588160/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829177 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829152.130853:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829152.130856:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787829152.130858:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f3a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572588160:192.168.203.129@tcp:38:kworker.0 00000100:00100000:2.0:1787829152.130862:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f392c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572588288:192.168.203.129@tcp:38:kworker.0 00000800:00000100:2.0:1787829152.130873:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787829152.130879:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829152/real 1787829152] req@ffff8cb999f392c0 x1874672572588288/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829177 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829152.130900:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f392c0 x1874672572588288/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829177 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829152.130906:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829152.130908:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787829152.130910:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f392c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572588288:192.168.203.129@tcp:38:kworker.0 00010000:00010000:0.0:1787829152.962448:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787829153.986389:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787829155.010384:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787829156.034456:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787829157.058457:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787829157.250337:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829157.250347:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787829157.250352:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829157.250362:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 2109 00000100:00080000:1.0:1787829157.250395:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829157.250403:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:1.0:1787829157.250442:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f38f00 x1874672572588416/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829157.250471:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829157.250474:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787829157.250475:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829157.250478:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 2109 00000100:00080000:1.0:1787829157.250480:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829157.250482:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:1.0:1787829157.250492:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787829157.250501:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f3b0c0 x1874672572588544/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829157.250509:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829157.250511:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1787829157.250526:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829157.250533:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 2109 00000100:00100000:2.0:1787829157.250535:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f38f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572588416:192.168.203.129@tcp:250:kworker.0 00000100:00080000:1.0:1787829157.250536:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829157.250543:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:1.0:1787829157.250548:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787829157.250559:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f3b480 x1874672572588672/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1787829157.250609:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787829157.250654:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829157/real 1787829157] req@ffff8cb999f38f00 x1874672572588416/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829182 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829157.250668:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f38f00 x1874672572588416/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829182 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829157.250678:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829157.250681:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787829157.250685:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f38f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572588416:192.168.203.129@tcp:250:kworker.0 00000100:00100000:2.0:1787829157.250690:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f3b0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572588544:192.168.203.129@tcp:38:kworker.0 00000800:00000100:2.0:1787829157.250706:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787829157.250716:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829157/real 1787829157] req@ffff8cb999f3b0c0 x1874672572588544/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829182 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829157.250724:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f3b0c0 x1874672572588544/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829182 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829157.250742:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829157.250745:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787829157.250747:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f3b0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572588544:192.168.203.129@tcp:38:kworker.0 00000100:00100000:2.0:1787829157.250775:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f3b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572588672:192.168.203.129@tcp:38:kworker.0 00000800:00000100:2.0:1787829157.250790:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787829157.250797:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829157/real 1787829157] req@ffff8cb999f3b480 x1874672572588672/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829182 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829157.250805:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f3b480 x1874672572588672/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829182 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829157.250815:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829157.250818:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787829157.250822:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f3b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572588672:192.168.203.129@tcp:38:kworker.0 00010000:00010000:0.0:1787829158.082405:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787829159.106447:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787829160.130427:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787829161.154536:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787829162.178584:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787829162.370453:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829162.370529:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787829162.370538:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829162.370557:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 2114 00000100:00080000:1.0:1787829162.370578:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829162.370595:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:1.0:1787829162.370662:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f38b40 x1874672572588800/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829162.370724:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829162.370732:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787829162.370737:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829162.370746:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 2114 00000100:00080000:1.0:1787829162.370755:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829162.370774:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:2.0:1787829162.370783:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f38b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572588800:192.168.203.129@tcp:250:kworker.0 00000008:00100000:1.0:1787829162.370796:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787829162.370828:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f39a40 x1874672572588928/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829162.370854:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829162.370862:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1787829162.370866:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829162.370874:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 2114 00000100:00080000:1.0:1787829162.370882:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000800:00000100:2.0:1787829162.370891:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00080000:1.0:1787829162.370899:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:1.0:1787829162.370912:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787829162.370938:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f39e00 x1874672572589056/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787829162.370961:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829162/real 1787829162] req@ffff8cb999f38b40 x1874672572588800/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829187 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829162.371004:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f38b40 x1874672572588800/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829187 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829162.371021:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829162.371028:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787829162.371046:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f38b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572588800:192.168.203.129@tcp:250:kworker.0 00000100:00100000:2.0:1787829162.371081:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f39a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572588928:192.168.203.129@tcp:38:kworker.0 00000800:00000100:2.0:1787829162.371111:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787829162.371134:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829162/real 1787829162] req@ffff8cb999f39a40 x1874672572588928/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829187 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829162.371157:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f39a40 x1874672572588928/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829187 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829162.371173:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829162.371179:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787829162.371186:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f39a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572588928:192.168.203.129@tcp:38:kworker.0 00000100:00100000:2.0:1787829162.371198:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f39e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572589056:192.168.203.129@tcp:38:kworker.0 00000800:00000100:2.0:1787829162.371236:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787829162.371256:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829162/real 1787829162] req@ffff8cb999f39e00 x1874672572589056/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829187 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829162.371311:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f39e00 x1874672572589056/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829187 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829162.371321:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829162.371324:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787829162.371328:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f39e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572589056:192.168.203.129@tcp:38:kworker.0 00010000:00010000:0.0:1787829163.202423:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787829164.226479:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787829165.250499:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787829166.274441:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787829167.298528:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787829167.490344:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829167.490355:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787829167.490361:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829167.490373:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 2119 00000100:00080000:3.0:1787829167.490387:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829167.490394:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:3.0:1787829167.490439:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb992317480 x1874672572589184/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787829167.490480:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829167.490484:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787829167.490485:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829167.490489:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 2119 00000100:00080000:3.0:1787829167.490502:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829167.490505:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:3.0:1787829167.490515:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787829167.490526:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb992314b40 x1874672572589312/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787829167.490561:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829167.490566:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1787829167.490569:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1787829167.490578:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb992317480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572589184:192.168.203.129@tcp:250:kworker.0 00000100:00080000:3.0:1787829167.490579:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 2119 00000100:00080000:3.0:1787829167.490590:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829167.490600:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:3.0:1787829167.490605:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787829167.490621:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb992314780 x1874672572589440/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1787829167.490680:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787829167.490770:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829167/real 1787829167] req@ffff8cb992317480 x1874672572589184/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829192 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829167.490796:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb992317480 x1874672572589184/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829192 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829167.490812:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829167.490819:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787829167.490824:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb992317480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572589184:192.168.203.129@tcp:250:kworker.0 00000100:00100000:2.0:1787829167.490855:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb992314b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572589312:192.168.203.129@tcp:38:kworker.0 00000800:00000100:2.0:1787829167.490886:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787829167.490904:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829167/real 1787829167] req@ffff8cb992314b40 x1874672572589312/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829192 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829167.490922:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb992314b40 x1874672572589312/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829192 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829167.490933:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829167.490938:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787829167.490943:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb992314b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572589312:192.168.203.129@tcp:38:kworker.0 00000100:00100000:2.0:1787829167.490951:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb992314780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572589440:192.168.203.129@tcp:38:kworker.0 00000800:00000100:2.0:1787829167.490978:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787829167.490994:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829167/real 1787829167] req@ffff8cb992314780 x1874672572589440/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829192 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829167.491012:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb992314780 x1874672572589440/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829192 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829167.491039:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829167.491044:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787829167.491049:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb992314780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572589440:192.168.203.129@tcp:38:kworker.0 00010000:00010000:0.0:1787829168.322589:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787829169.346402:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787829170.370507:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787829171.394504:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787829172.418481:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787829172.610372:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829172.610380:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787829172.610384:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829172.610392:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 2124 00000100:00080000:3.0:1787829172.610402:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829172.610423:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:3.0:1787829172.610462:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb992316580 x1874672572589568/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787829172.610491:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829172.610495:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787829172.610496:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829172.610499:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 2124 00000100:00080000:3.0:1787829172.610504:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829172.610510:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:3.0:1787829172.610531:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787829172.610543:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb992316580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572589568:192.168.203.129@tcp:250:kworker.0 00000100:00100000:3.0:1787829172.610547:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb992314780 x1874672572589696/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787829172.610570:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829172.610574:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1787829172.610576:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829172.610584:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 2124 00000100:00080000:3.0:1787829172.610589:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829172.610592:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:3.0:1787829172.610600:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787829172.610614:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb992314b40 x1874672572589824/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1787829172.610642:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787829172.610687:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829172/real 1787829172] req@ffff8cb992316580 x1874672572589568/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829197 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829172.610702:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb992316580 x1874672572589568/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829197 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829172.610712:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829172.610715:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787829172.610718:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb992316580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572589568:192.168.203.129@tcp:250:kworker.0 00000100:00100000:1.0:1787829172.610743:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb992314780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572589696:192.168.203.129@tcp:38:kworker.0 00000800:00000100:1.0:1787829172.610760:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787829172.610769:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829172/real 1787829172] req@ffff8cb992314780 x1874672572589696/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829197 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829172.610777:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb992314780 x1874672572589696/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829197 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829172.610784:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829172.610787:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787829172.610790:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb992314780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572589696:192.168.203.129@tcp:38:kworker.0 00000100:00100000:1.0:1787829172.610794:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb992314b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572589824:192.168.203.129@tcp:38:kworker.0 00000800:00000100:1.0:1787829172.610806:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787829172.610826:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829172/real 1787829172] req@ffff8cb992314b40 x1874672572589824/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829197 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829172.610835:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb992314b40 x1874672572589824/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829197 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829172.610843:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829172.610845:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787829172.610848:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb992314b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572589824:192.168.203.129@tcp:38:kworker.0 00010000:00010000:0.0:1787829173.442484:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787829174.466557:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787829175.490415:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787829176.514516:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787829177.538411:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787829177.730363:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829177.730384:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787829177.730388:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829177.730409:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 2129 00000100:00080000:2.0:1787829177.730420:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829177.730429:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:2.0:1787829177.730463:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8f480 x1874672572589952/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829177.730491:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829177.730495:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787829177.730498:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829177.730502:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 2129 00000100:00080000:2.0:1787829177.730507:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829177.730512:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:1.0:1787829177.730518:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572589952:192.168.203.129@tcp:250:kworker.0 00000008:00100000:2.0:1787829177.730523:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787829177.730540:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8c000 x1874672572590080/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829177.730554:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829177.730557:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1787829177.730573:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829177.730579:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 2129 00000100:00080000:2.0:1787829177.730584:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829177.730589:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:2.0:1787829177.730595:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787829177.730611:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8cb40 x1874672572590208/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787829177.730612:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572590080:192.168.203.129@tcp:38:kworker.0 00000100:00100000:1.0:1787829177.730651:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8cb40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572590208:192.168.203.129@tcp:38:kworker.0 00010000:00010000:0.0:1787829178.562425:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787829179.586418:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787829180.610686:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:0.0:1787829180.802596:0:2384:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.129:988 00000400:00000100:0.0:1787829180.802653:0:2384:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.129@tcp at host 192.168.203.129 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:0.0:1787829180.802675:0:2384:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.129@tcp 00000800:00000100:0.0:1787829180.802721:0:2384:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.129@tcp 00000800:00000100:0.0:1787829180.802736:0:2384:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.129@tcp 00000100:00100000:1.0:1787829180.802763:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829177/real 1787829180] req@ffff8cb998a8cb40 x1874672572590208/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829202 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829180.802795:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8cb40 x1874672572590208/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829202 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829180.802817:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829180.802825:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787829180.802829:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8cb40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572590208:192.168.203.129@tcp:38:kworker.0 00000100:00100000:1.0:1787829180.802840:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829177/real 1787829180] req@ffff8cb998a8c000 x1874672572590080/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829202 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829180.802855:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8c000 x1874672572590080/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829202 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829180.802885:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829180.802890:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787829180.802895:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572590080:192.168.203.129@tcp:38:kworker.0 00000100:00100000:1.0:1787829180.802903:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829177/real 1787829180] req@ffff8cb998a8f480 x1874672572589952/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829202 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829180.802918:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8f480 x1874672572589952/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829202 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829180.802928:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829180.802931:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787829180.802935:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572589952:192.168.203.129@tcp:250:kworker.0 00010000:00010000:1.0:1787829181.634510:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787829182.658498:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787829182.850405:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829182.850414:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787829182.850419:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829182.850429:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 2135 00000100:00080000:3.0:1787829182.850440:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829182.850447:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:3.0:1787829182.850488:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb992317480 x1874672572590336/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787829182.850522:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829182.850527:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787829182.850529:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829182.850533:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 2135 00000100:00080000:3.0:1787829182.850559:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829182.850565:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:3.0:1787829182.850578:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787829182.850582:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb992317480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572590336:192.168.203.129@tcp:250:kworker.0 00000100:00100000:3.0:1787829182.850596:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb992315e00 x1874672572590464/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787829182.850610:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829182.850614:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1787829182.850617:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829182.850621:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 2135 00000100:00080000:3.0:1787829182.850626:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829182.850631:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:3.0:1787829182.850637:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787829182.850649:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb992316940 x1874672572590592/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1787829182.850675:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787829182.850736:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829182/real 1787829182] req@ffff8cb992317480 x1874672572590336/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829207 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829182.850759:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb992317480 x1874672572590336/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829207 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829182.850771:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829182.850776:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787829182.850780:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb992317480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572590336:192.168.203.129@tcp:250:kworker.0 00000100:00100000:1.0:1787829182.850824:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb992315e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572590464:192.168.203.129@tcp:38:kworker.0 00000800:00000100:1.0:1787829182.850851:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787829182.850865:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829182/real 1787829182] req@ffff8cb992315e00 x1874672572590464/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829207 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829182.850877:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb992315e00 x1874672572590464/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829207 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829182.850885:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829182.850889:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787829182.850892:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb992315e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572590464:192.168.203.129@tcp:38:kworker.0 00000100:00100000:1.0:1787829182.850897:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb992316940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572590592:192.168.203.129@tcp:38:kworker.0 00000800:00000100:1.0:1787829182.850918:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787829182.850930:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829182/real 1787829182] req@ffff8cb992316940 x1874672572590592/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829207 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829182.850940:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb992316940 x1874672572590592/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829207 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829182.850948:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829182.850952:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787829182.850956:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb992316940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572590592:192.168.203.129@tcp:38:kworker.0 00010000:00010000:1.0:1787829183.682746:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787829184.706577:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787829185.730507:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787829186.754685:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787829187.778528:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787829187.970412:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829187.970422:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787829187.970427:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829187.970437:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 2140 00000100:00080000:1.0:1787829187.970449:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829187.970456:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:1.0:1787829187.970497:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f3b840 x1874672572590720/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829187.970527:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829187.970534:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787829187.970536:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829187.970538:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 2140 00000100:00080000:1.0:1787829187.970541:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829187.970544:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:1.0:1787829187.970554:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787829187.970566:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f39e00 x1874672572590848/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829187.970575:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829187.970579:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00100000:2.0:1787829187.970581:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f3b840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572590720:192.168.203.129@tcp:250:kworker.0 00000100:00080000:1.0:1787829187.970582:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829187.970590:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 2140 00000100:00080000:1.0:1787829187.970593:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829187.970596:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:1.0:1787829187.970616:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787829187.970628:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f39a40 x1874672572590976/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1787829187.970657:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787829187.970710:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829187/real 1787829187] req@ffff8cb999f3b840 x1874672572590720/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829212 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829187.970728:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f3b840 x1874672572590720/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829212 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829187.970738:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829187.970742:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787829187.970745:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f3b840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572590720:192.168.203.129@tcp:250:kworker.0 00000100:00100000:2.0:1787829187.970775:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f39e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572590848:192.168.203.129@tcp:38:kworker.0 00000800:00000100:2.0:1787829187.970792:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787829187.970802:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829187/real 1787829187] req@ffff8cb999f39e00 x1874672572590848/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829212 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829187.970811:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f39e00 x1874672572590848/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829212 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829187.970828:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829187.970831:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787829187.970833:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f39e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572590848:192.168.203.129@tcp:38:kworker.0 00000100:00100000:2.0:1787829187.970838:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f39a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572590976:192.168.203.129@tcp:38:kworker.0 00000800:00000100:2.0:1787829187.970854:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787829187.970861:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829187/real 1787829187] req@ffff8cb999f39a40 x1874672572590976/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829212 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829187.970869:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f39a40 x1874672572590976/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829212 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829187.970874:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829187.970877:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787829187.970879:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f39a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572590976:192.168.203.129@tcp:38:kworker.0 00010000:00010000:1.0:1787829188.802605:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787829189.826464:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787829190.850517:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787829191.874459:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787829192.898499:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787829193.090530:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829193.090548:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787829193.090554:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829193.090571:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 2145 00000100:00080000:1.0:1787829193.090589:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829193.090601:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:1.0:1787829193.090677:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f38b40 x1874672572591104/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829193.090722:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829193.090727:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787829193.090729:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829193.090734:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 2145 00000100:00080000:1.0:1787829193.090738:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829193.090744:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:1.0:1787829193.090758:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787829193.090778:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f3b480 x1874672572591232/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787829193.090790:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f38b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572591104:192.168.203.129@tcp:250:kworker.0 00000100:00080000:1.0:1787829193.090827:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829193.090831:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1787829193.090833:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829193.090839:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 2145 00000100:00080000:1.0:1787829193.090843:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829193.090848:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:1.0:1787829193.090855:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787829193.090868:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f3b0c0 x1874672572591360/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1787829193.090880:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787829193.090927:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829193/real 1787829193] req@ffff8cb999f38b40 x1874672572591104/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829218 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829193.090942:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f38b40 x1874672572591104/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829218 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829193.090951:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829193.090955:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787829193.090971:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f38b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572591104:192.168.203.129@tcp:250:kworker.0 00000100:00100000:2.0:1787829193.090995:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f3b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572591232:192.168.203.129@tcp:38:kworker.0 00000800:00000100:2.0:1787829193.091015:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787829193.091025:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829193/real 1787829193] req@ffff8cb999f3b480 x1874672572591232/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829218 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829193.091034:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f3b480 x1874672572591232/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829218 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829193.091040:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829193.091043:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787829193.091046:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f3b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572591232:192.168.203.129@tcp:38:kworker.0 00000100:00100000:2.0:1787829193.091050:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f3b0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572591360:192.168.203.129@tcp:38:kworker.0 00000800:00000100:2.0:1787829193.091061:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787829193.091068:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829193/real 1787829193] req@ffff8cb999f3b0c0 x1874672572591360/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829218 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829193.091076:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f3b0c0 x1874672572591360/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829218 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829193.091082:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829193.091084:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787829193.091086:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f3b0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572591360:192.168.203.129@tcp:38:kworker.0 00010000:00010000:1.0:1787829193.922467:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787829194.946447:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787829195.970490:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787829196.994515:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787829198.018417:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787829198.210371:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829198.210384:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787829198.210389:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829198.210418:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 2150 00000100:00080000:1.0:1787829198.210430:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829198.210438:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:1.0:1787829198.210493:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f38f00 x1874672572591488/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829198.210529:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829198.210540:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787829198.210576:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829198.210581:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 2150 00000100:00080000:1.0:1787829198.210587:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829198.210592:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:1.0:1787829198.210602:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787829198.210622:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f392c0 x1874672572591616/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787829198.210624:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f38f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572591488:192.168.203.129@tcp:250:kworker.0 00000100:00080000:1.0:1787829198.210648:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829198.210652:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1787829198.210655:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829198.210663:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 2150 00000100:00080000:1.0:1787829198.210668:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829198.210673:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:1.0:1787829198.210679:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787829198.210694:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f3a580 x1874672572591744/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1787829198.210723:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787829198.210813:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829198/real 1787829198] req@ffff8cb999f38f00 x1874672572591488/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829223 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829198.210835:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f38f00 x1874672572591488/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829223 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829198.210848:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829198.210852:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787829198.210857:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f38f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572591488:192.168.203.129@tcp:250:kworker.0 00000100:00100000:2.0:1787829198.210898:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f392c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572591616:192.168.203.129@tcp:38:kworker.0 00000800:00000100:2.0:1787829198.210934:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787829198.210954:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829198/real 1787829198] req@ffff8cb999f392c0 x1874672572591616/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829223 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829198.210967:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f392c0 x1874672572591616/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829223 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829198.210975:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829198.210978:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787829198.210981:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f392c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572591616:192.168.203.129@tcp:38:kworker.0 00000100:00100000:2.0:1787829198.210987:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f3a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572591744:192.168.203.129@tcp:38:kworker.0 00000800:00000100:2.0:1787829198.211008:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787829198.211017:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829198/real 1787829198] req@ffff8cb999f3a580 x1874672572591744/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829223 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829198.211028:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f3a580 x1874672572591744/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829223 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829198.211048:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829198.211051:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787829198.211054:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f3a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572591744:192.168.203.129@tcp:38:kworker.0 00010000:00010000:1.0:1787829199.042413:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787829200.066411:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787829201.090422:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787829202.114506:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787829203.138483:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787829203.330430:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829203.330441:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787829203.330446:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829203.330456:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 2155 00000100:00080000:2.0:1787829203.330469:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829203.330477:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:2.0:1787829203.330518:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8d680 x1874672572591872/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829203.330563:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829203.330567:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787829203.330568:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829203.330572:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 2155 00000100:00080000:2.0:1787829203.330575:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829203.330578:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:2.0:1787829203.330587:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787829203.330616:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8e580 x1874672572592000/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829203.330628:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829203.330632:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00100000:1.0:1787829203.330641:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8d680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572591872:192.168.203.129@tcp:250:kworker.0 00000100:00080000:2.0:1787829203.330681:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829203.330690:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 2155 00000100:00080000:2.0:1787829203.330694:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829203.330698:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:2.0:1787829203.330704:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787829203.330716:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8cf00 x1874672572592128/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1787829203.330742:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787829203.330808:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829203/real 1787829203] req@ffff8cb998a8d680 x1874672572591872/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829228 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829203.330836:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8d680 x1874672572591872/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829228 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829203.330878:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829203.330885:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787829203.330892:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8d680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572591872:192.168.203.129@tcp:250:kworker.0 00000100:00100000:1.0:1787829203.330925:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8e580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572592000:192.168.203.129@tcp:38:kworker.0 00000800:00000100:1.0:1787829203.330962:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787829203.330984:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829203/real 1787829203] req@ffff8cb998a8e580 x1874672572592000/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829228 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829203.331004:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8e580 x1874672572592000/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829228 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829203.331017:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829203.331023:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787829203.331028:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8e580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572592000:192.168.203.129@tcp:38:kworker.0 00000100:00100000:1.0:1787829203.331037:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8cf00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572592128:192.168.203.129@tcp:38:kworker.0 00000800:00000100:1.0:1787829203.331068:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787829203.331086:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829203/real 1787829203] req@ffff8cb998a8cf00 x1874672572592128/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829228 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829203.331106:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8cf00 x1874672572592128/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829228 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829203.331134:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829203.331140:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787829203.331145:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8cf00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572592128:192.168.203.129@tcp:38:kworker.0 00010000:00010000:1.0:1787829204.162432:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787829205.186443:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787829206.210574:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787829207.234442:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787829208.258488:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787829208.450420:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829208.450432:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787829208.450438:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829208.450452:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 2160 00000100:00080000:3.0:1787829208.450465:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829208.450481:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:3.0:1787829208.450528:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb992315a40 x1874672572592256/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787829208.450564:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829208.450577:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787829208.450579:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829208.450585:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 2160 00000100:00080000:3.0:1787829208.450590:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829208.450596:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:3.0:1787829208.450610:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787829208.450623:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb992315a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572592256:192.168.203.129@tcp:250:kworker.0 00000100:00100000:3.0:1787829208.450629:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb992314f00 x1874672572592384/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787829208.450646:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829208.450650:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1787829208.450653:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829208.450658:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 2160 00000100:00080000:3.0:1787829208.450662:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829208.450667:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:3.0:1787829208.450672:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787829208.450687:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb9923161c0 x1874672572592512/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1787829208.450723:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787829208.450791:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829208/real 1787829208] req@ffff8cb992315a40 x1874672572592256/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829233 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829208.450814:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb992315a40 x1874672572592256/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829233 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829208.450824:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829208.450828:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787829208.450831:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb992315a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572592256:192.168.203.129@tcp:250:kworker.0 00000100:00100000:1.0:1787829208.450860:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb992314f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572592384:192.168.203.129@tcp:38:kworker.0 00000800:00000100:1.0:1787829208.450888:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787829208.450910:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829208/real 1787829208] req@ffff8cb992314f00 x1874672572592384/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829233 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829208.450924:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb992314f00 x1874672572592384/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829233 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829208.450933:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829208.450937:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787829208.450941:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb992314f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572592384:192.168.203.129@tcp:38:kworker.0 00000100:00100000:1.0:1787829208.450957:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb9923161c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572592512:192.168.203.129@tcp:38:kworker.0 00000800:00000100:1.0:1787829208.450974:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787829208.450991:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829208/real 1787829208] req@ffff8cb9923161c0 x1874672572592512/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829233 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829208.451001:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb9923161c0 x1874672572592512/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829233 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829208.451007:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829208.451009:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787829208.451011:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb9923161c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572592512:192.168.203.129@tcp:38:kworker.0 00010000:00010000:0.0:1787829209.282460:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787829210.306459:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787829211.330492:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787829212.354501:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787829213.378432:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787829213.570359:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829213.570370:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787829213.570375:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829213.570388:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 2165 00000100:00080000:3.0:1787829213.570401:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829213.570408:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:3.0:1787829213.570458:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb9923170c0 x1874672572592640/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787829213.570510:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829213.570513:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787829213.570515:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829213.570520:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 2165 00000100:00080000:3.0:1787829213.570525:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829213.570530:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:3.0:1787829213.570544:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787829213.570559:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb9923170c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572592640:192.168.203.129@tcp:250:kworker.0 00000100:00100000:3.0:1787829213.570565:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb9923161c0 x1874672572592768/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787829213.570576:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829213.570579:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1787829213.570581:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829213.570584:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 2165 00000100:00080000:3.0:1787829213.570588:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829213.570590:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:3.0:1787829213.570595:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787829213.570605:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb992314f00 x1874672572592896/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1787829213.570647:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787829213.570715:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829213/real 1787829213] req@ffff8cb9923170c0 x1874672572592640/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829238 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829213.570734:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb9923170c0 x1874672572592640/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829238 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829213.570744:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829213.570748:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787829213.570763:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb9923170c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572592640:192.168.203.129@tcp:250:kworker.0 00000100:00100000:1.0:1787829213.570792:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb9923161c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572592768:192.168.203.129@tcp:38:kworker.0 00000800:00000100:1.0:1787829213.570820:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787829213.570833:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829213/real 1787829213] req@ffff8cb9923161c0 x1874672572592768/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829238 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829213.570844:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb9923161c0 x1874672572592768/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829238 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829213.570852:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829213.570855:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787829213.570858:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb9923161c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572592768:192.168.203.129@tcp:38:kworker.0 00000100:00100000:1.0:1787829213.570864:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb992314f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572592896:192.168.203.129@tcp:38:kworker.0 00000800:00000100:1.0:1787829213.570878:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787829213.570887:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829213/real 1787829213] req@ffff8cb992314f00 x1874672572592896/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829238 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829213.570909:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb992314f00 x1874672572592896/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829238 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829213.570917:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829213.570919:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787829213.570922:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb992314f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572592896:192.168.203.129@tcp:38:kworker.0 00010000:00010000:1.0:1787829214.402476:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787829215.426430:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787829216.450450:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787829217.474411:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787829218.498419:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787829218.690429:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829218.690440:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787829218.690444:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829218.690455:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 2170 00000100:00080000:2.0:1787829218.690467:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829218.690474:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:2.0:1787829218.690519:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8c780 x1874672572593024/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829218.690548:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829218.690552:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787829218.690553:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829218.690556:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 2170 00000100:00080000:2.0:1787829218.690560:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829218.690562:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:2.0:1787829218.690574:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787829218.690590:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8f0c0 x1874672572593152/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787829218.690602:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8c780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572593024:192.168.203.129@tcp:250:kworker.0 00000100:00080000:2.0:1787829218.690635:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829218.690638:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1787829218.690640:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829218.690644:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 2170 00000100:00080000:2.0:1787829218.690648:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829218.690651:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:2.0:1787829218.690657:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787829218.690668:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8fc00 x1874672572593280/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1787829218.690687:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787829218.690754:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829218/real 1787829218] req@ffff8cb998a8c780 x1874672572593024/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829243 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829218.690785:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8c780 x1874672572593024/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829243 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829218.690795:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829218.690798:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787829218.690802:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8c780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572593024:192.168.203.129@tcp:250:kworker.0 00000100:00100000:1.0:1787829218.690835:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8f0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572593152:192.168.203.129@tcp:38:kworker.0 00000800:00000100:1.0:1787829218.690856:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787829218.690867:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829218/real 1787829218] req@ffff8cb998a8f0c0 x1874672572593152/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829243 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829218.690877:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8f0c0 x1874672572593152/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829243 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829218.690883:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829218.690885:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787829218.690888:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8f0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572593152:192.168.203.129@tcp:38:kworker.0 00000100:00100000:1.0:1787829218.690892:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8fc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572593280:192.168.203.129@tcp:38:kworker.0 00000800:00000100:1.0:1787829218.690905:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787829218.690912:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829218/real 1787829218] req@ffff8cb998a8fc00 x1874672572593280/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829243 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829218.690920:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8fc00 x1874672572593280/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829243 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829218.690937:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829218.690939:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787829218.690942:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8fc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572593280:192.168.203.129@tcp:38:kworker.0 00010000:00010000:0.0:1787829219.522470:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787829220.546424:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787829221.570420:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787829222.594480:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787829223.618394:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787829223.810351:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829223.810366:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787829223.810369:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829223.810378:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 2175 00000100:00080000:2.0:1787829223.810388:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829223.810394:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:2.0:1787829223.810443:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8e1c0 x1874672572593408/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829223.810473:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829223.810477:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787829223.810479:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829223.810483:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 2175 00000100:00080000:2.0:1787829223.810488:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829223.810493:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:2.0:1787829223.810518:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787829223.810519:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8e1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572593408:192.168.203.129@tcp:250:kworker.0 00000100:00100000:2.0:1787829223.810539:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8da40 x1874672572593536/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829223.810553:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829223.810558:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1787829223.810560:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829223.810564:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 2175 00000100:00080000:2.0:1787829223.810569:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829223.810574:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:2.0:1787829223.810580:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000800:00000100:1.0:1787829223.810587:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787829223.810595:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8de00 x1874672572593664/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787829223.810635:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829223/real 1787829223] req@ffff8cb998a8e1c0 x1874672572593408/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829248 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829223.810649:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8e1c0 x1874672572593408/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829248 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829223.810657:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829223.810660:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787829223.810663:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8e1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572593408:192.168.203.129@tcp:250:kworker.0 00000100:00100000:1.0:1787829223.810684:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8da40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572593536:192.168.203.129@tcp:38:kworker.0 00000800:00000100:1.0:1787829223.810707:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787829223.810721:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829223/real 1787829223] req@ffff8cb998a8da40 x1874672572593536/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829248 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829223.810735:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8da40 x1874672572593536/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829248 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829223.810751:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829223.810753:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787829223.810756:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8da40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572593536:192.168.203.129@tcp:38:kworker.0 00000100:00100000:1.0:1787829223.810761:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8de00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572593664:192.168.203.129@tcp:38:kworker.0 00000800:00000100:1.0:1787829223.810781:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787829223.810794:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829223/real 1787829223] req@ffff8cb998a8de00 x1874672572593664/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829248 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829223.810807:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8de00 x1874672572593664/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829248 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829223.810816:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829223.810819:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787829223.810823:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8de00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572593664:192.168.203.129@tcp:38:kworker.0 00010000:00010000:1.0:1787829224.642449:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787829225.666388:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787829226.690428:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787829227.714465:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787829228.738470:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787829228.930379:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829228.930389:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787829228.930394:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829228.930404:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 2181 00000100:00080000:2.0:1787829228.930417:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829228.930426:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:2.0:1787829228.930478:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8c3c0 x1874672572593792/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829228.930506:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829228.930510:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787829228.930511:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829228.930514:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 2181 00000100:00080000:2.0:1787829228.930535:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829228.930539:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:2.0:1787829228.930549:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787829228.930559:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8d2c0 x1874672572593920/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829228.930567:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829228.930570:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1787829228.930572:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829228.930575:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 2181 00000100:00080000:2.0:1787829228.930577:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829228.930580:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:2.0:1787829228.930583:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787829228.930595:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8c3c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572593792:192.168.203.129@tcp:250:kworker.0 00000100:00100000:2.0:1787829228.930595:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8de00 x1874672572594048/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1787829228.930671:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787829228.930718:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829228/real 1787829228] req@ffff8cb998a8c3c0 x1874672572593792/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829253 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829228.930733:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8c3c0 x1874672572593792/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829253 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829228.930749:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829228.930753:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787829228.930756:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8c3c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572593792:192.168.203.129@tcp:250:kworker.0 00000100:00100000:1.0:1787829228.930762:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8d2c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572593920:192.168.203.129@tcp:38:kworker.0 00000800:00000100:1.0:1787829228.930780:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787829228.930789:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829228/real 1787829228] req@ffff8cb998a8d2c0 x1874672572593920/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829253 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829228.930797:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8d2c0 x1874672572593920/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829253 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829228.930803:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829228.930804:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787829228.930806:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8d2c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572593920:192.168.203.129@tcp:38:kworker.0 00000100:00100000:1.0:1787829228.930830:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8de00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572594048:192.168.203.129@tcp:38:kworker.0 00000800:00000100:1.0:1787829228.930842:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787829228.930849:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829228/real 1787829228] req@ffff8cb998a8de00 x1874672572594048/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829253 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829228.930857:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8de00 x1874672572594048/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829253 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829228.930862:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829228.930864:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787829228.930865:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8de00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572594048:192.168.203.129@tcp:38:kworker.0 00010000:00010000:1.0:1787829229.762445:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787829230.786422:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787829231.810459:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787829232.834487:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787829233.858480:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787829234.050358:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829234.050369:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787829234.050374:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829234.050398:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 2186 00000100:00080000:2.0:1787829234.050413:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829234.050423:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:2.0:1787829234.050472:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8da40 x1874672572594176/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829234.050503:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829234.050525:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787829234.050527:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829234.050531:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 2186 00000100:00080000:2.0:1787829234.050535:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829234.050539:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:2.0:1787829234.050549:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787829234.050556:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8da40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572594176:192.168.203.129@tcp:250:kworker.0 00000100:00100000:2.0:1787829234.050564:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8e1c0 x1874672572594304/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829234.050574:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829234.050576:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1787829234.050578:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829234.050581:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 2186 00000100:00080000:2.0:1787829234.050583:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829234.050585:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:2.0:1787829234.050589:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787829234.050599:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8fc00 x1874672572594432/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1787829234.050647:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787829234.050691:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829234/real 1787829234] req@ffff8cb998a8da40 x1874672572594176/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829259 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829234.050705:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8da40 x1874672572594176/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829259 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829234.050714:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829234.050717:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787829234.050720:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8da40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572594176:192.168.203.129@tcp:250:kworker.0 00000100:00100000:1.0:1787829234.050744:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8e1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572594304:192.168.203.129@tcp:38:kworker.0 00000800:00000100:1.0:1787829234.050761:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787829234.050769:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829234/real 1787829234] req@ffff8cb998a8e1c0 x1874672572594304/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829259 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829234.050777:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8e1c0 x1874672572594304/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829259 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829234.050783:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829234.050784:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787829234.050787:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8e1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572594304:192.168.203.129@tcp:38:kworker.0 00000100:00100000:1.0:1787829234.050798:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8fc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572594432:192.168.203.129@tcp:38:kworker.0 00000800:00000100:1.0:1787829234.050812:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787829234.050820:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829234/real 1787829234] req@ffff8cb998a8fc00 x1874672572594432/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829259 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829234.050828:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8fc00 x1874672572594432/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829259 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829234.050833:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829234.050836:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787829234.050838:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8fc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572594432:192.168.203.129@tcp:38:kworker.0 00010000:00010000:1.0:1787829234.882406:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787829235.906477:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787829236.930425:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787829237.954420:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787829238.978475:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787829239.170440:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829239.170453:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787829239.170458:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829239.170471:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 2191 00000100:00080000:3.0:1787829239.170507:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829239.170515:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:3.0:1787829239.170561:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb992315a40 x1874672572594560/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787829239.170593:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829239.170598:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787829239.170601:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829239.170605:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 2191 00000100:00080000:3.0:1787829239.170609:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829239.170614:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:3.0:1787829239.170628:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787829239.170643:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb992316940 x1874672572594688/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787829239.170648:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb992315a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572594560:192.168.203.129@tcp:250:kworker.0 00000100:00080000:3.0:1787829239.170663:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829239.170677:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1787829239.170683:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829239.170690:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 2191 00000100:00080000:3.0:1787829239.170693:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829239.170697:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:3.0:1787829239.170702:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787829239.170712:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb992315e00 x1874672572594816/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1787829239.170737:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787829239.170795:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829239/real 1787829239] req@ffff8cb992315a40 x1874672572594560/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829264 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829239.170814:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb992315a40 x1874672572594560/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829264 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829239.170826:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829239.170831:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787829239.170835:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb992315a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572594560:192.168.203.129@tcp:250:kworker.0 00000100:00100000:1.0:1787829239.170891:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb992316940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572594688:192.168.203.129@tcp:38:kworker.0 00000800:00000100:1.0:1787829239.170919:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787829239.170934:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829239/real 1787829239] req@ffff8cb992316940 x1874672572594688/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829264 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829239.170964:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb992316940 x1874672572594688/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829264 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829239.170974:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829239.170977:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787829239.170982:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb992316940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572594688:192.168.203.129@tcp:38:kworker.0 00000100:00100000:1.0:1787829239.170989:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb992315e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572594816:192.168.203.129@tcp:38:kworker.0 00000800:00000100:1.0:1787829239.171011:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787829239.171024:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829239/real 1787829239] req@ffff8cb992315e00 x1874672572594816/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829264 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829239.171037:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb992315e00 x1874672572594816/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829264 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829239.171047:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829239.171050:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787829239.171054:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb992315e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572594816:192.168.203.129@tcp:38:kworker.0 00010000:00010000:1.0:1787829240.002464:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787829241.026401:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787829242.050474:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787829243.074405:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787829244.098517:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787829244.290430:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829244.290467:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787829244.290473:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829244.290506:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 2196 00000100:00080000:3.0:1787829244.290522:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829244.290533:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:3.0:1787829244.290580:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb992317480 x1874672572594944/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787829244.290615:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829244.290619:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787829244.290622:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829244.290629:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 2196 00000100:00080000:3.0:1787829244.290635:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829244.290641:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:2.0:1787829244.290655:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb992317480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572594944:192.168.203.129@tcp:250:kworker.0 00000008:00100000:3.0:1787829244.290656:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787829244.290681:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb992314b40 x1874672572595072/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787829244.290696:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829244.290703:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1787829244.290706:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829244.290714:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 2196 00000100:00080000:3.0:1787829244.290725:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829244.290733:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:3.0:1787829244.290742:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787829244.290763:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb992314780 x1874672572595200/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787829244.290794:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb992314b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572595072:192.168.203.129@tcp:38:kworker.0 00000100:00100000:0.0:1787829244.291062:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb992314780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572595200:192.168.203.129@tcp:38:kworker.0 00010000:00010000:1.0:1787829245.122615:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787829246.146538:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787829247.170483:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:1.0:1787829247.362619:0:2381:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.129:988 00000400:00000100:1.0:1787829247.362672:0:2381:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.129@tcp at host 192.168.203.129 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:1.0:1787829247.362693:0:2381:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.129@tcp 00000800:00000100:1.0:1787829247.362759:0:2381:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.129@tcp 00000800:00000100:1.0:1787829247.362773:0:2381:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.129@tcp 00000100:00100000:0.0:1787829247.362802:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829244/real 1787829247] req@ffff8cb992314780 x1874672572595200/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829269 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829247.362835:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb992314780 x1874672572595200/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829269 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829247.362851:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829247.362858:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787829247.362864:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb992314780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572595200:192.168.203.129@tcp:38:kworker.0 00000100:00100000:0.0:1787829247.362875:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829244/real 1787829247] req@ffff8cb992314b40 x1874672572595072/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829269 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829247.362916:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb992314b40 x1874672572595072/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829269 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829247.362926:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829247.362931:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787829247.362935:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb992314b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572595072:192.168.203.129@tcp:38:kworker.0 00000100:00100000:0.0:1787829247.362943:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829244/real 1787829247] req@ffff8cb992317480 x1874672572594944/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829269 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829247.362958:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb992317480 x1874672572594944/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829269 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829247.362967:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829247.362971:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787829247.362993:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb992317480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572594944:192.168.203.129@tcp:250:kworker.0 00010000:00010000:0.0:1787829248.194714:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829249.218970:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787829249.410356:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829249.410364:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787829249.410368:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829249.410376:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 2201 00000100:00080000:3.0:1787829249.410385:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829249.410391:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:3.0:1787829249.410419:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb992316580 x1874672572595328/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787829249.410441:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829249.410444:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787829249.410445:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829249.410448:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 2201 00000100:00080000:3.0:1787829249.410450:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829249.410453:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:3.0:1787829249.410462:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787829249.410471:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb992317480 x1874672572595456/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787829249.410481:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1787829249.410484:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb992316580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572595328:192.168.203.129@tcp:250:kworker.0 00000100:00080000:3.0:1787829249.410495:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1787829249.410498:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829249.410503:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 2201 00000100:00080000:3.0:1787829249.410508:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829249.410512:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:3.0:1787829249.410535:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000800:00000100:0.0:1787829249.410549:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:3.0:1787829249.410553:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb992314b40 x1874672572595584/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787829249.410594:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829249/real 1787829249] req@ffff8cb992316580 x1874672572595328/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829274 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829249.410614:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb992316580 x1874672572595328/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829274 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829249.410624:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829249.410629:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787829249.410633:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb992316580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572595328:192.168.203.129@tcp:250:kworker.0 00000100:00100000:0.0:1787829249.410657:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb992317480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572595456:192.168.203.129@tcp:38:kworker.0 00000800:00000100:0.0:1787829249.410680:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787829249.410694:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829249/real 1787829249] req@ffff8cb992317480 x1874672572595456/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829274 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829249.410707:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb992317480 x1874672572595456/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829274 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829249.410717:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829249.410720:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787829249.410724:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb992317480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572595456:192.168.203.129@tcp:38:kworker.0 00000100:00100000:0.0:1787829249.410731:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb992314b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572595584:192.168.203.129@tcp:38:kworker.0 00000800:00000100:0.0:1787829249.410766:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787829249.410778:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829249/real 1787829249] req@ffff8cb992314b40 x1874672572595584/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829274 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829249.410790:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb992314b40 x1874672572595584/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829274 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829249.410799:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829249.410802:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787829249.410806:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb992314b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572595584:192.168.203.129@tcp:38:kworker.0 00010000:00010000:0.0:1787829250.243009:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829251.266553:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829252.290878:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787829253.314692:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829254.338500:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787829254.530340:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829254.530348:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787829254.530352:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829254.530360:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 2206 00000100:00080000:2.0:1787829254.530370:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829254.530375:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:2.0:1787829254.530408:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8f0c0 x1874672572595712/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829254.530432:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829254.530435:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787829254.530436:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829254.530439:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 2206 00000100:00080000:2.0:1787829254.530441:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829254.530443:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:2.0:1787829254.530451:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787829254.530459:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8c780 x1874672572595840/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829254.530467:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829254.530469:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1787829254.530470:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829254.530473:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 2206 00000100:00080000:2.0:1787829254.530475:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829254.530482:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:0.0:1787829254.530483:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8f0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572595712:192.168.203.129@tcp:250:kworker.0 00000008:00100000:2.0:1787829254.530485:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787829254.530509:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8cf00 x1874672572595968/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1787829254.530544:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787829254.530577:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829254/real 1787829254] req@ffff8cb998a8f0c0 x1874672572595712/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829279 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829254.530589:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8f0c0 x1874672572595712/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829279 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829254.530598:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829254.530600:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787829254.530603:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8f0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572595712:192.168.203.129@tcp:250:kworker.0 00000100:00100000:0.0:1787829254.530608:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8c780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572595840:192.168.203.129@tcp:38:kworker.0 00000800:00000100:0.0:1787829254.530621:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787829254.530629:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829254/real 1787829254] req@ffff8cb998a8c780 x1874672572595840/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829279 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829254.530648:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8c780 x1874672572595840/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829279 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829254.530654:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829254.530656:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787829254.530658:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8c780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572595840:192.168.203.129@tcp:38:kworker.0 00000100:00100000:0.0:1787829254.530680:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8cf00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572595968:192.168.203.129@tcp:38:kworker.0 00000800:00000100:0.0:1787829254.530703:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787829254.530714:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829254/real 1787829254] req@ffff8cb998a8cf00 x1874672572595968/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829279 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829254.530727:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8cf00 x1874672572595968/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829279 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829254.530734:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829254.530737:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787829254.530740:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8cf00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572595968:192.168.203.129@tcp:38:kworker.0 00010000:00010000:2.0:1787829255.362636:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787829256.386804:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829257.410423:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829258.434462:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787829259.458735:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787829259.650492:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829259.650509:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787829259.650516:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829259.650532:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 2211 00000100:00080000:3.0:1787829259.650549:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829259.650559:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:3.0:1787829259.650651:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb992314780 x1874672572596096/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787829259.650692:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829259.650698:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787829259.650701:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829259.650706:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 2211 00000100:00080000:3.0:1787829259.650710:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829259.650716:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:3.0:1787829259.650732:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787829259.650750:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb992315e00 x1874672572596224/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787829259.650759:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb992314780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572596096:192.168.203.129@tcp:250:kworker.0 00000100:00080000:3.0:1787829259.650776:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829259.650782:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1787829259.650785:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829259.650791:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 2211 00000100:00080000:3.0:1787829259.650860:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829259.650868:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:3.0:1787829259.650876:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787829259.650896:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb992316940 x1874672572596352/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1787829259.650912:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787829259.650985:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829259/real 1787829259] req@ffff8cb992314780 x1874672572596096/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829284 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829259.651011:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb992314780 x1874672572596096/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829284 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829259.651066:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829259.651073:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787829259.651081:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb992314780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572596096:192.168.203.129@tcp:250:kworker.0 00000100:00100000:0.0:1787829259.651159:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb992315e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572596224:192.168.203.129@tcp:38:kworker.0 00000800:00000100:0.0:1787829259.651207:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787829259.651228:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829259/real 1787829259] req@ffff8cb992315e00 x1874672572596224/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829284 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829259.651244:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb992315e00 x1874672572596224/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829284 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829259.651254:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829259.651259:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787829259.651263:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb992315e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572596224:192.168.203.129@tcp:38:kworker.0 00000100:00100000:0.0:1787829259.651271:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb992316940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572596352:192.168.203.129@tcp:38:kworker.0 00000800:00000100:0.0:1787829259.651330:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787829259.651344:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829259/real 1787829259] req@ffff8cb992316940 x1874672572596352/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829284 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829259.651359:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb992316940 x1874672572596352/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829284 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829259.651369:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829259.651373:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787829259.651377:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb992316940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572596352:192.168.203.129@tcp:38:kworker.0 00010000:00010000:2.0:1787829260.482418:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829261.506522:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787829262.530370:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829263.554443:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829264.578432:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787829264.770374:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829264.770386:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787829264.770392:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829264.770417:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 2216 00000100:00080000:3.0:1787829264.770432:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829264.770440:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:3.0:1787829264.770493:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb992315a40 x1874672572596480/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787829264.770529:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829264.770533:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787829264.770536:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829264.770539:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 2216 00000100:00080000:3.0:1787829264.770543:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829264.770547:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:3.0:1787829264.770560:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787829264.770573:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb992314f00 x1874672572596608/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787829264.770586:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1787829264.770603:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb992315a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572596480:192.168.203.129@tcp:250:kworker.0 00000100:00080000:3.0:1787829264.770615:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1787829264.770618:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829264.770625:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 2216 00000100:00080000:3.0:1787829264.770631:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829264.770637:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:3.0:1787829264.770643:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787829264.770657:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb9923161c0 x1874672572596736/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1787829264.770675:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787829264.770727:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829264/real 1787829264] req@ffff8cb992315a40 x1874672572596480/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829289 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829264.770743:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb992315a40 x1874672572596480/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829289 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829264.770752:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829264.770756:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787829264.770759:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb992315a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572596480:192.168.203.129@tcp:250:kworker.0 00000100:00100000:0.0:1787829264.770798:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb992314f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572596608:192.168.203.129@tcp:38:kworker.0 00000800:00000100:0.0:1787829264.770817:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787829264.770826:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829264/real 1787829264] req@ffff8cb992314f00 x1874672572596608/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829289 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829264.770834:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb992314f00 x1874672572596608/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829289 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829264.770840:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829264.770842:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787829264.770844:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb992314f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572596608:192.168.203.129@tcp:38:kworker.0 00000100:00100000:0.0:1787829264.770849:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb9923161c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572596736:192.168.203.129@tcp:38:kworker.0 00000800:00000100:0.0:1787829264.770861:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787829264.770883:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829264/real 1787829264] req@ffff8cb9923161c0 x1874672572596736/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829289 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829264.770891:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb9923161c0 x1874672572596736/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829289 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829264.770897:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829264.770899:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787829264.770901:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb9923161c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572596736:192.168.203.129@tcp:38:kworker.0 00010000:00010000:1.0:1787829265.602426:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829266.626519:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829267.650459:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787829268.674551:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829269.698472:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787829269.890383:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829269.890399:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787829269.890404:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829269.890425:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 2222 00000100:00080000:3.0:1787829269.890441:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829269.890450:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:3.0:1787829269.890502:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb9923170c0 x1874672572596864/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787829269.890535:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829269.890538:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787829269.890540:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829269.890543:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 2222 00000100:00080000:3.0:1787829269.890546:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829269.890549:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:3.0:1787829269.890563:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787829269.890576:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb9923161c0 x1874672572596992/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787829269.890586:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb9923170c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572596864:192.168.203.129@tcp:250:kworker.0 00000100:00080000:3.0:1787829269.890602:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829269.890606:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1787829269.890608:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829269.890612:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 2222 00000100:00080000:3.0:1787829269.890615:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829269.890618:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:3.0:1787829269.890623:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787829269.890634:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb992314f00 x1874672572597120/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1787829269.890655:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787829269.890710:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829269/real 1787829269] req@ffff8cb9923170c0 x1874672572596864/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829294 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829269.890725:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb9923170c0 x1874672572596864/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829294 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829269.890750:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829269.890754:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787829269.890757:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb9923170c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572596864:192.168.203.129@tcp:250:kworker.0 00000100:00100000:0.0:1787829269.890786:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb9923161c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572596992:192.168.203.129@tcp:38:kworker.0 00000800:00000100:0.0:1787829269.890805:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787829269.890815:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829269/real 1787829269] req@ffff8cb9923161c0 x1874672572596992/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829294 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829269.890823:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb9923161c0 x1874672572596992/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829294 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829269.890840:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829269.890843:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787829269.890845:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb9923161c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572596992:192.168.203.129@tcp:38:kworker.0 00000100:00100000:0.0:1787829269.890850:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb992314f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572597120:192.168.203.129@tcp:38:kworker.0 00000800:00000100:0.0:1787829269.890864:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787829269.890871:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829269/real 1787829269] req@ffff8cb992314f00 x1874672572597120/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829294 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829269.890878:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb992314f00 x1874672572597120/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829294 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829269.890884:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829269.890886:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787829269.890888:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb992314f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572597120:192.168.203.129@tcp:38:kworker.0 00010000:00010000:2.0:1787829270.722430:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787829271.746483:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829272.770411:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829273.794459:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787829274.818451:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787829275.010380:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829275.010389:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787829275.010394:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829275.010404:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 2227 00000100:00080000:2.0:1787829275.010424:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829275.010431:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:2.0:1787829275.010475:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8e580 x1874672572597248/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829275.010501:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829275.010505:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787829275.010508:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829275.010511:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 2227 00000100:00080000:2.0:1787829275.010534:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829275.010539:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:2.0:1787829275.010550:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787829275.010551:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8e580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572597248:192.168.203.129@tcp:250:kworker.0 00000100:00100000:2.0:1787829275.010562:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8d680 x1874672572597376/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829275.010572:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829275.010576:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1787829275.010577:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829275.010581:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 2227 00000100:00080000:2.0:1787829275.010584:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829275.010588:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:2.0:1787829275.010592:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787829275.010601:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8f480 x1874672572597504/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1787829275.010625:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787829275.010678:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829275/real 1787829275] req@ffff8cb998a8e580 x1874672572597248/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829300 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829275.010706:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8e580 x1874672572597248/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829300 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829275.010716:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829275.010719:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787829275.010728:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8e580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572597248:192.168.203.129@tcp:250:kworker.0 00000100:00100000:0.0:1787829275.010755:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8d680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572597376:192.168.203.129@tcp:38:kworker.0 00000800:00000100:0.0:1787829275.010775:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787829275.010784:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829275/real 1787829275] req@ffff8cb998a8d680 x1874672572597376/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829300 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829275.010792:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8d680 x1874672572597376/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829300 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829275.010798:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829275.010800:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787829275.010803:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8d680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572597376:192.168.203.129@tcp:38:kworker.0 00000100:00100000:0.0:1787829275.010806:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572597504:192.168.203.129@tcp:38:kworker.0 00000800:00000100:0.0:1787829275.010817:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787829275.010822:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829275/real 1787829275] req@ffff8cb998a8f480 x1874672572597504/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829300 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829275.010830:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8f480 x1874672572597504/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829300 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829275.010835:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829275.010837:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787829275.010839:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572597504:192.168.203.129@tcp:38:kworker.0 00010000:00010000:2.0:1787829275.842448:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829276.866455:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787829277.890487:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829278.914486:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829279.938480:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787829280.130470:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829280.130482:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787829280.130486:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829280.130518:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 2232 00000100:00080000:3.0:1787829280.130531:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829280.130542:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:3.0:1787829280.130581:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb992315a40 x1874672572597632/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787829280.130614:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829280.130620:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787829280.130623:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829280.130630:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 2232 00000100:00080000:3.0:1787829280.130636:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829280.130642:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:0.0:1787829280.130714:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb992315a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572597632:192.168.203.129@tcp:250:kworker.0 00000008:00100000:3.0:1787829280.130719:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787829280.130739:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb992316940 x1874672572597760/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787829280.130757:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829280.130761:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1787829280.130764:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829280.130771:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 2232 00000100:00080000:3.0:1787829280.130777:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000800:00000100:0.0:1787829280.130782:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00080000:3.0:1787829280.130786:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:3.0:1787829280.130793:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787829280.130831:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829280/real 1787829280] req@ffff8cb992315a40 x1874672572597632/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829305 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829280.130847:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb992315a40 x1874672572597632/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829305 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829280.130856:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829280.130860:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787829280.130865:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb992315a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572597632:192.168.203.129@tcp:250:kworker.0 00000100:00100000:0.0:1787829280.130889:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb992316940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572597760:192.168.203.129@tcp:38:kworker.0 00000800:00000100:0.0:1787829280.130908:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:3.0:1787829280.130916:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb992315e00 x1874672572597888/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787829280.130920:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829280/real 1787829280] req@ffff8cb992316940 x1874672572597760/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829305 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829280.130949:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb992316940 x1874672572597760/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829305 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829280.130957:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829280.130961:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787829280.130964:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb992316940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572597760:192.168.203.129@tcp:38:kworker.0 00000100:00100000:0.0:1787829280.130974:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb992315e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572597888:192.168.203.129@tcp:38:kworker.0 00000800:00000100:0.0:1787829280.130986:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787829280.130993:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829280/real 1787829280] req@ffff8cb992315e00 x1874672572597888/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829305 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829280.131000:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb992315e00 x1874672572597888/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829305 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829280.131015:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829280.131018:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787829280.131020:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb992315e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572597888:192.168.203.129@tcp:38:kworker.0 00010000:00010000:1.0:1787829280.962476:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829281.986478:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829283.010416:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787829284.034382:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829285.058403:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787829285.250373:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829285.250383:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787829285.250388:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829285.250398:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 2237 00000100:00080000:3.0:1787829285.250408:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829285.250415:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:3.0:1787829285.250459:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb992314780 x1874672572598016/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787829285.250497:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829285.250503:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787829285.250505:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829285.250510:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 2237 00000100:00080000:3.0:1787829285.250539:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829285.250548:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:0.0:1787829285.250550:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb992314780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572598016:192.168.203.129@tcp:250:kworker.0 00000008:00100000:3.0:1787829285.250560:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787829285.250581:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb992314b40 x1874672572598144/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787829285.250595:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829285.250598:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1787829285.250601:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829285.250605:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 2237 00000100:00080000:3.0:1787829285.250618:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829285.250640:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:3.0:1787829285.250647:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000800:00000100:0.0:1787829285.250651:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:3.0:1787829285.250667:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb992317480 x1874672572598272/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787829285.250705:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829285/real 1787829285] req@ffff8cb992314780 x1874672572598016/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829310 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829285.250725:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb992314780 x1874672572598016/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829310 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829285.250736:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829285.250740:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787829285.250744:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb992314780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572598016:192.168.203.129@tcp:250:kworker.0 00000100:00100000:0.0:1787829285.250770:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb992314b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572598144:192.168.203.129@tcp:38:kworker.0 00000800:00000100:0.0:1787829285.250793:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787829285.250805:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829285/real 1787829285] req@ffff8cb992314b40 x1874672572598144/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829310 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829285.250816:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb992314b40 x1874672572598144/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829310 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829285.250825:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829285.250828:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787829285.250831:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb992314b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572598144:192.168.203.129@tcp:38:kworker.0 00000100:00100000:0.0:1787829285.250848:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb992317480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572598272:192.168.203.129@tcp:38:kworker.0 00000800:00000100:0.0:1787829285.250868:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787829285.250878:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829285/real 1787829285] req@ffff8cb992317480 x1874672572598272/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829310 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829285.250889:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb992317480 x1874672572598272/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829310 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829285.250897:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829285.250900:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787829285.250903:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb992317480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572598272:192.168.203.129@tcp:38:kworker.0 00010000:00010000:2.0:1787829286.082426:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787829287.106381:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829288.130407:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829289.154449:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787829290.178394:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787829290.370444:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829290.370455:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787829290.370459:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829290.370466:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 2242 00000100:00080000:1.0:1787829290.370496:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829290.370504:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:1.0:1787829290.370553:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f383c0 x1874672572598400/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829290.370583:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829290.370637:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787829290.370639:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829290.370642:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 2242 00000100:00080000:1.0:1787829290.370646:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829290.370649:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:1.0:1787829290.370658:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787829290.370670:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f3a580 x1874672572598528/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787829290.370679:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f383c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572598400:192.168.203.129@tcp:250:kworker.0 00000100:00080000:1.0:1787829290.370698:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829290.370705:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1787829290.370707:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829290.370711:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 2242 00000100:00080000:1.0:1787829290.370713:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829290.370716:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:1.0:1787829290.370720:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787829290.370729:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f392c0 x1874672572598656/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1787829290.370783:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787829290.370848:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829290/real 1787829290] req@ffff8cb999f383c0 x1874672572598400/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829315 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829290.370899:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f383c0 x1874672572598400/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829315 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829290.370907:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829290.370910:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787829290.370913:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f383c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572598400:192.168.203.129@tcp:250:kworker.0 00000100:00100000:0.0:1787829290.370932:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f3a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572598528:192.168.203.129@tcp:38:kworker.0 00000800:00000100:0.0:1787829290.370974:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787829290.370993:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829290/real 1787829290] req@ffff8cb999f3a580 x1874672572598528/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829315 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829290.371041:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f3a580 x1874672572598528/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829315 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829290.371055:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829290.371059:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787829290.371063:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f3a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572598528:192.168.203.129@tcp:38:kworker.0 00000100:00100000:0.0:1787829290.371084:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f392c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572598656:192.168.203.129@tcp:38:kworker.0 00000800:00000100:0.0:1787829290.371107:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787829290.371116:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829290/real 1787829290] req@ffff8cb999f392c0 x1874672572598656/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829315 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829290.371123:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f392c0 x1874672572598656/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829315 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829290.371128:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829290.371131:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787829290.371133:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f392c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572598656:192.168.203.129@tcp:38:kworker.0 00010000:00010000:2.0:1787829291.202503:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829292.226386:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787829293.250417:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829294.274435:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829295.298464:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787829295.490438:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829295.490448:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787829295.490453:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829295.490477:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 2247 00000100:00080000:3.0:1787829295.490489:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829295.490495:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:3.0:1787829295.490543:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb992316580 x1874672572598784/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787829295.490587:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829295.490625:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787829295.490627:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829295.490631:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 2247 00000100:00080000:3.0:1787829295.490635:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829295.490639:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:0.0:1787829295.490647:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb992316580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572598784:192.168.203.129@tcp:250:kworker.0 00000008:00100000:3.0:1787829295.490650:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787829295.490666:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb992317480 x1874672572598912/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787829295.490676:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829295.490679:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1787829295.490680:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829295.490683:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 2247 00000100:00080000:3.0:1787829295.490686:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829295.490688:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:3.0:1787829295.490691:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787829295.490700:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb992314b40 x1874672572599040/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1787829295.490721:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787829295.490765:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829295/real 1787829295] req@ffff8cb992316580 x1874672572598784/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829320 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829295.490780:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb992316580 x1874672572598784/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829320 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829295.490801:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829295.490804:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787829295.490808:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb992316580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572598784:192.168.203.129@tcp:250:kworker.0 00000100:00100000:0.0:1787829295.490832:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb992317480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572598912:192.168.203.129@tcp:38:kworker.0 00000800:00000100:0.0:1787829295.490862:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787829295.490871:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829295/real 1787829295] req@ffff8cb992317480 x1874672572598912/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829320 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829295.490880:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb992317480 x1874672572598912/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829320 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829295.490886:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829295.490888:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787829295.490890:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb992317480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572598912:192.168.203.129@tcp:38:kworker.0 00000100:00100000:0.0:1787829295.490894:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb992314b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572599040:192.168.203.129@tcp:38:kworker.0 00000800:00000100:0.0:1787829295.490906:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787829295.490916:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829295/real 1787829295] req@ffff8cb992314b40 x1874672572599040/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829320 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829295.490929:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb992314b40 x1874672572599040/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829320 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829295.490939:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829295.490943:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787829295.490946:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb992314b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572599040:192.168.203.129@tcp:38:kworker.0 00010000:00010000:1.0:1787829296.322566:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829297.346419:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829298.370512:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787829299.394497:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829300.418456:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787829300.610342:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829300.610353:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787829300.610357:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829300.610367:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 2252 00000100:00080000:3.0:1787829300.610377:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829300.610384:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:3.0:1787829300.610415:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb992314780 x1874672572599168/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787829300.610448:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829300.610451:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787829300.610452:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829300.610455:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 2252 00000100:00080000:3.0:1787829300.610458:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829300.610460:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:3.0:1787829300.610469:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787829300.610479:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb992315e00 x1874672572599296/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787829300.610487:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1787829300.610490:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb992314780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572599168:192.168.203.129@tcp:250:kworker.0 00000100:00080000:3.0:1787829300.610491:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1787829300.610494:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829300.610502:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 2252 00000100:00080000:3.0:1787829300.610504:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829300.610506:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:3.0:1787829300.610509:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787829300.610531:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb992316940 x1874672572599424/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1787829300.610590:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787829300.610641:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829300/real 1787829300] req@ffff8cb992314780 x1874672572599168/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829325 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829300.610662:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb992314780 x1874672572599168/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829325 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829300.610671:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829300.610674:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787829300.610678:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb992314780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572599168:192.168.203.129@tcp:250:kworker.0 00000100:00100000:0.0:1787829300.610700:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb992315e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572599296:192.168.203.129@tcp:38:kworker.0 00000800:00000100:0.0:1787829300.610725:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787829300.610740:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829300/real 1787829300] req@ffff8cb992315e00 x1874672572599296/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829325 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829300.610754:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb992315e00 x1874672572599296/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829325 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829300.610761:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829300.610764:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787829300.610766:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb992315e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572599296:192.168.203.129@tcp:38:kworker.0 00000100:00100000:0.0:1787829300.610771:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb992316940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572599424:192.168.203.129@tcp:38:kworker.0 00000800:00000100:0.0:1787829300.610788:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787829300.610795:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829300/real 1787829300] req@ffff8cb992316940 x1874672572599424/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829325 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829300.610810:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb992316940 x1874672572599424/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829325 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829300.610816:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829300.610818:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787829300.610820:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb992316940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572599424:192.168.203.129@tcp:38:kworker.0 00010000:00010000:2.0:1787829301.442396:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787829302.466491:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829303.490460:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829304.514451:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787829305.538407:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787829305.730323:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829305.730334:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787829305.730337:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829305.730346:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 2257 00000100:00080000:3.0:1787829305.730368:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829305.730375:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:3.0:1787829305.730404:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb992315a40 x1874672572599552/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787829305.730426:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829305.730429:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787829305.730431:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829305.730433:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 2257 00000100:00080000:3.0:1787829305.730435:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829305.730437:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:3.0:1787829305.730445:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787829305.730454:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb992314f00 x1874672572599680/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787829305.730462:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829305.730466:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00100000:0.0:1787829305.730467:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb992315a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572599552:192.168.203.129@tcp:250:kworker.0 00000100:00080000:3.0:1787829305.730468:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829305.730476:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 2257 00000100:00080000:3.0:1787829305.730479:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829305.730497:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:3.0:1787829305.730501:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787829305.730511:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb9923161c0 x1874672572599808/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1787829305.730556:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787829305.730586:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829305/real 1787829305] req@ffff8cb992315a40 x1874672572599552/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829330 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829305.730599:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb992315a40 x1874672572599552/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829330 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829305.730607:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829305.730610:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787829305.730613:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb992315a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572599552:192.168.203.129@tcp:250:kworker.0 00000100:00100000:0.0:1787829305.730634:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb992314f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572599680:192.168.203.129@tcp:38:kworker.0 00000800:00000100:0.0:1787829305.730649:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787829305.730656:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829305/real 1787829305] req@ffff8cb992314f00 x1874672572599680/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829330 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829305.730664:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb992314f00 x1874672572599680/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829330 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829305.730679:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829305.730682:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787829305.730684:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb992314f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572599680:192.168.203.129@tcp:38:kworker.0 00000100:00100000:0.0:1787829305.730689:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb9923161c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572599808:192.168.203.129@tcp:38:kworker.0 00000800:00000100:0.0:1787829305.730702:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787829305.730708:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829305/real 1787829305] req@ffff8cb9923161c0 x1874672572599808/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829330 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829305.730716:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb9923161c0 x1874672572599808/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829330 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829305.730721:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829305.730723:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787829305.730725:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb9923161c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572599808:192.168.203.129@tcp:38:kworker.0 00010000:00010000:2.0:1787829306.562492:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829307.586453:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787829308.610457:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829309.634401:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829310.658438:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787829310.850349:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829310.850377:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787829310.850381:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829310.850392:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 2263 00000100:00080000:3.0:1787829310.850404:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829310.850410:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:3.0:1787829310.850465:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb9923170c0 x1874672572599936/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787829310.850500:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829310.850503:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787829310.850505:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829310.850508:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 2263 00000100:00080000:3.0:1787829310.850512:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829310.850515:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:3.0:1787829310.850526:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787829310.850539:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb9923161c0 x1874672572600064/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787829310.850548:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829310.850551:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1787829310.850553:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829310.850556:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 2263 00000100:00080000:3.0:1787829310.850559:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1787829310.850562:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb9923170c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572599936:192.168.203.129@tcp:250:kworker.0 00000100:00080000:3.0:1787829310.850564:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:3.0:1787829310.850573:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787829310.850580:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb992314f00 x1874672572600192/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787829310.850690:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb9923161c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572600064:192.168.203.129@tcp:38:kworker.0 00000100:00100000:0.0:1787829310.850712:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb992314f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572600192:192.168.203.129@tcp:38:kworker.0 00010000:00010000:1.0:1787829311.682400:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829312.706492:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829313.730745:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:1.0:1787829313.922654:0:2382:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.129:988 00000400:00000100:1.0:1787829313.922718:0:2382:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.129@tcp at host 192.168.203.129 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:1.0:1787829313.922761:0:2382:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.129@tcp 00000800:00000100:1.0:1787829313.922814:0:2382:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.129@tcp 00000800:00000100:1.0:1787829313.922826:0:2382:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.129@tcp 00000100:00100000:0.0:1787829313.922945:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829310/real 1787829313] req@ffff8cb9923161c0 x1874672572600064/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829335 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829313.922998:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb9923161c0 x1874672572600064/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829335 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829313.923039:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829313.923046:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787829313.923051:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb9923161c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572600064:192.168.203.129@tcp:38:kworker.0 00000100:00100000:0.0:1787829313.923060:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829310/real 1787829313] req@ffff8cb992314f00 x1874672572600192/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829335 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829313.923075:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb992314f00 x1874672572600192/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829335 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829313.923084:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829313.923088:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787829313.923091:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb992314f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572600192:192.168.203.129@tcp:38:kworker.0 00000100:00100000:0.0:1787829313.923099:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829310/real 1787829313] req@ffff8cb9923170c0 x1874672572599936/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829335 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829313.923111:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb9923170c0 x1874672572599936/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829335 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829313.923119:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829313.923122:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787829313.923125:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb9923170c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572599936:192.168.203.129@tcp:250:kworker.0 00010000:00010000:2.0:1787829314.754615:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829315.778603:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787829315.970443:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829315.970454:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787829315.970459:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829315.970473:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 2268 00000100:00080000:3.0:1787829315.970487:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829315.970494:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:3.0:1787829315.970541:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb992315a40 x1874672572600320/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787829315.970585:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829315.970591:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787829315.970593:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829315.970601:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 2268 00000100:00080000:3.0:1787829315.970609:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829315.970615:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:3.0:1787829315.970631:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787829315.970648:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb992316940 x1874672572600448/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787829315.970658:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb992315a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572600320:192.168.203.129@tcp:250:kworker.0 00000100:00080000:3.0:1787829315.970664:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829315.970669:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1787829315.970671:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829315.970678:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 2268 00000100:00080000:3.0:1787829315.970682:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829315.970688:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:3.0:1787829315.970717:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787829315.970737:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb992315e00 x1874672572600576/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1787829315.970782:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787829315.970861:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829315/real 1787829315] req@ffff8cb992315a40 x1874672572600320/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829340 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829315.970897:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb992315a40 x1874672572600320/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829340 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829315.970909:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829315.970914:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787829315.970919:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb992315a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572600320:192.168.203.129@tcp:250:kworker.0 00000100:00100000:0.0:1787829315.970954:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb992316940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572600448:192.168.203.129@tcp:38:kworker.0 00000800:00000100:0.0:1787829315.970980:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787829315.970995:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829315/real 1787829315] req@ffff8cb992316940 x1874672572600448/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829340 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829315.971008:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb992316940 x1874672572600448/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829340 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829315.971019:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829315.971023:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787829315.971027:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb992316940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572600448:192.168.203.129@tcp:38:kworker.0 00000100:00100000:0.0:1787829315.971034:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb992315e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572600576:192.168.203.129@tcp:38:kworker.0 00000800:00000100:0.0:1787829315.971053:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787829315.971065:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829315/real 1787829315] req@ffff8cb992315e00 x1874672572600576/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829340 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829315.971077:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb992315e00 x1874672572600576/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829340 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829315.971111:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829315.971115:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787829315.971118:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb992315e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572600576:192.168.203.129@tcp:38:kworker.0 00010000:00010000:2.0:1787829316.802629:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829317.826721:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829318.850481:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829319.874767:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829320.898666:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787829321.090387:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829321.090396:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787829321.090399:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829321.090407:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 2273 00000100:00080000:1.0:1787829321.090419:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829321.090427:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:1.0:1787829321.090464:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f38f00 x1874672572600704/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829321.090496:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829321.090501:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787829321.090503:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829321.090507:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 2273 00000100:00080000:1.0:1787829321.090510:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829321.090513:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:1.0:1787829321.090522:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787829321.090534:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f3b0c0 x1874672572600832/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787829321.090539:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f38f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572600704:192.168.203.129@tcp:250:kworker.0 00000100:00080000:1.0:1787829321.090549:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829321.090552:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1787829321.090553:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829321.090557:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 2273 00000100:00080000:1.0:1787829321.090589:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829321.090596:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000800:00000100:0.0:1787829321.090597:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000008:00100000:1.0:1787829321.090607:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787829321.090624:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f3b480 x1874672572600960/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787829321.090635:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829321/real 1787829321] req@ffff8cb999f38f00 x1874672572600704/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829346 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829321.090661:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f38f00 x1874672572600704/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829346 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829321.090670:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829321.090675:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787829321.090679:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f38f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572600704:192.168.203.129@tcp:250:kworker.0 00000100:00100000:0.0:1787829321.090704:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f3b0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572600832:192.168.203.129@tcp:38:kworker.0 00000800:00000100:0.0:1787829321.090738:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787829321.090752:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829321/real 1787829321] req@ffff8cb999f3b0c0 x1874672572600832/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829346 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829321.090765:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f3b0c0 x1874672572600832/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829346 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829321.090778:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829321.090781:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787829321.090783:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f3b0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572600832:192.168.203.129@tcp:38:kworker.0 00000100:00100000:0.0:1787829321.090794:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f3b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572600960:192.168.203.129@tcp:38:kworker.0 00000800:00000100:0.0:1787829321.090816:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787829321.090829:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829321/real 1787829321] req@ffff8cb999f3b480 x1874672572600960/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829346 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829321.090842:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f3b480 x1874672572600960/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829346 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829321.090850:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829321.090854:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787829321.090857:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f3b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572600960:192.168.203.129@tcp:38:kworker.0 00010000:00010000:2.0:1787829321.922587:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829322.946600:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829323.970449:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829324.994442:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829326.018431:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787829326.210377:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829326.210385:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787829326.210389:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829326.210397:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 2278 00000100:00080000:3.0:1787829326.210406:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829326.210412:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:3.0:1787829326.210442:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb992314780 x1874672572601088/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787829326.210470:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829326.210472:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787829326.210474:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829326.210477:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 2278 00000100:00080000:3.0:1787829326.210479:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829326.210481:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:3.0:1787829326.210488:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787829326.210497:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb992314b40 x1874672572601216/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787829326.210504:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829326.210507:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1787829326.210508:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829326.210510:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 2278 00000100:00100000:0.0:1787829326.210519:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb992314780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572601088:192.168.203.129@tcp:250:kworker.0 00000100:00080000:3.0:1787829326.210528:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829326.210533:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:3.0:1787829326.210536:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787829326.210546:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb992317480 x1874672572601344/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1787829326.210587:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787829326.210624:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829326/real 1787829326] req@ffff8cb992314780 x1874672572601088/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829351 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829326.210639:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb992314780 x1874672572601088/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829351 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829326.210648:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829326.210651:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787829326.210654:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb992314780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572601088:192.168.203.129@tcp:250:kworker.0 00000100:00100000:0.0:1787829326.210659:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb992314b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572601216:192.168.203.129@tcp:38:kworker.0 00000800:00000100:0.0:1787829326.210674:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787829326.210682:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829326/real 1787829326] req@ffff8cb992314b40 x1874672572601216/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829351 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829326.210698:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb992314b40 x1874672572601216/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829351 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829326.210703:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829326.210706:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787829326.210708:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb992314b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572601216:192.168.203.129@tcp:38:kworker.0 00000100:00100000:0.0:1787829326.210727:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb992317480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572601344:192.168.203.129@tcp:38:kworker.0 00000800:00000100:0.0:1787829326.210740:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787829326.210746:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829326/real 1787829326] req@ffff8cb992317480 x1874672572601344/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829351 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829326.210754:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb992317480 x1874672572601344/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829351 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829326.210759:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829326.210762:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787829326.210764:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb992317480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572601344:192.168.203.129@tcp:38:kworker.0 00010000:00010000:2.0:1787829327.042397:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829328.066395:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829329.090407:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829330.114394:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829331.138462:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787829331.330396:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829331.330407:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787829331.330412:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829331.330423:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 2283 00000100:00080000:1.0:1787829331.330438:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829331.330445:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:1.0:1787829331.330490:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f38b40 x1874672572601472/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829331.330517:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829331.330522:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787829331.330524:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829331.330528:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 2283 00000100:00080000:1.0:1787829331.330532:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829331.330537:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:1.0:1787829331.330550:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787829331.330567:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f39a40 x1874672572601600/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787829331.330571:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f38b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572601472:192.168.203.129@tcp:250:kworker.0 00000100:00080000:1.0:1787829331.330588:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829331.330610:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1787829331.330613:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829331.330619:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 2283 00000100:00080000:1.0:1787829331.330639:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829331.330645:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000800:00000100:0.0:1787829331.330648:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000008:00100000:1.0:1787829331.330656:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787829331.330674:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f39e00 x1874672572601728/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787829331.330705:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829331/real 1787829331] req@ffff8cb999f38b40 x1874672572601472/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829356 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829331.330726:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f38b40 x1874672572601472/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829356 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829331.330751:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829331.330755:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787829331.330759:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f38b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572601472:192.168.203.129@tcp:250:kworker.0 00000100:00100000:0.0:1787829331.330788:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f39a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572601600:192.168.203.129@tcp:38:kworker.0 00000800:00000100:0.0:1787829331.330813:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787829331.330827:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829331/real 1787829331] req@ffff8cb999f39a40 x1874672572601600/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829356 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829331.330839:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f39a40 x1874672572601600/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829356 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829331.330845:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829331.330847:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787829331.330849:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f39a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572601600:192.168.203.129@tcp:38:kworker.0 00000100:00100000:0.0:1787829331.330853:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f39e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572601728:192.168.203.129@tcp:38:kworker.0 00000800:00000100:0.0:1787829331.330869:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787829331.330876:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829331/real 1787829331] req@ffff8cb999f39e00 x1874672572601728/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829356 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829331.330884:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f39e00 x1874672572601728/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829356 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829331.330889:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829331.330891:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787829331.330892:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f39e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572601728:192.168.203.129@tcp:38:kworker.0 00010000:00010000:2.0:1787829332.162393:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829333.186407:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829334.210405:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829335.234494:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829336.258453:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787829336.450414:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829336.450429:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787829336.450435:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829336.450448:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 2288 00000100:00080000:3.0:1787829336.450460:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829336.450467:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:3.0:1787829336.450513:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb992316580 x1874672572601856/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787829336.450539:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829336.450543:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787829336.450544:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829336.450547:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 2288 00000100:00080000:3.0:1787829336.450550:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829336.450554:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:3.0:1787829336.450564:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787829336.450577:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb992316580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572601856:192.168.203.129@tcp:250:kworker.0 00000100:00100000:3.0:1787829336.450578:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb992317480 x1874672572601984/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787829336.450595:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829336.450616:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1787829336.450617:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829336.450621:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 2288 00000100:00080000:3.0:1787829336.450624:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829336.450627:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:3.0:1787829336.450631:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787829336.450642:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb992314b40 x1874672572602112/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1787829336.450664:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787829336.450726:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829336/real 1787829336] req@ffff8cb992316580 x1874672572601856/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829361 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829336.450745:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb992316580 x1874672572601856/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829361 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829336.450754:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829336.450758:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787829336.450761:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb992316580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572601856:192.168.203.129@tcp:250:kworker.0 00000100:00100000:0.0:1787829336.450791:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb992317480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572601984:192.168.203.129@tcp:38:kworker.0 00000800:00000100:0.0:1787829336.450817:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787829336.450831:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829336/real 1787829336] req@ffff8cb992317480 x1874672572601984/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829361 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829336.450840:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb992317480 x1874672572601984/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829361 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829336.450847:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829336.450849:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787829336.450852:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb992317480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572601984:192.168.203.129@tcp:38:kworker.0 00000100:00100000:0.0:1787829336.450856:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb992314b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572602112:192.168.203.129@tcp:38:kworker.0 00000800:00000100:0.0:1787829336.450879:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787829336.450887:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829336/real 1787829336] req@ffff8cb992314b40 x1874672572602112/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829361 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829336.450903:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb992314b40 x1874672572602112/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829361 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829336.450909:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829336.450911:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787829336.450913:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb992314b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572602112:192.168.203.129@tcp:38:kworker.0 00010000:00010000:2.0:1787829337.282474:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829338.306418:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829339.330447:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829340.354432:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829341.378431:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787829341.570421:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829341.570436:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787829341.570441:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829341.570454:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 2293 00000100:00080000:2.0:1787829341.570469:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829341.570476:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:2.0:1787829341.570531:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8c000 x1874672572602240/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829341.570557:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829341.570560:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787829341.570561:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829341.570564:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 2293 00000100:00080000:2.0:1787829341.570567:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829341.570570:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:2.0:1787829341.570580:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787829341.570589:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8cb40 x1874672572602368/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787829341.570607:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572602240:192.168.203.129@tcp:250:kworker.0 00000100:00080000:2.0:1787829341.570624:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829341.570627:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1787829341.570628:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829341.570632:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 2293 00000100:00080000:2.0:1787829341.570636:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829341.570639:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:2.0:1787829341.570642:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787829341.570653:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8e940 x1874672572602496/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1787829341.570695:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787829341.570748:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829341/real 1787829341] req@ffff8cb998a8c000 x1874672572602240/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829366 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829341.570770:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8c000 x1874672572602240/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829366 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829341.570795:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829341.570800:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787829341.570805:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572602240:192.168.203.129@tcp:250:kworker.0 00000100:00100000:0.0:1787829341.570835:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8cb40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572602368:192.168.203.129@tcp:38:kworker.0 00000800:00000100:0.0:1787829341.570859:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787829341.570874:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829341/real 1787829341] req@ffff8cb998a8cb40 x1874672572602368/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829366 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829341.570889:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8cb40 x1874672572602368/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829366 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829341.570914:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829341.570918:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787829341.570923:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8cb40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572602368:192.168.203.129@tcp:38:kworker.0 00000100:00100000:0.0:1787829341.570930:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8e940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572602496:192.168.203.129@tcp:38:kworker.0 00000800:00000100:0.0:1787829341.570954:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787829341.570968:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829341/real 1787829341] req@ffff8cb998a8e940 x1874672572602496/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829366 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829341.570992:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8e940 x1874672572602496/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829366 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829341.571002:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829341.571007:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787829341.571010:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8e940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572602496:192.168.203.129@tcp:38:kworker.0 00010000:00010000:2.0:1787829342.402455:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829343.426425:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787829344.450468:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829345.474443:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829346.498434:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787829346.690388:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829346.690399:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787829346.690403:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829346.690413:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 2298 00000100:00080000:3.0:1787829346.690424:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829346.690432:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:3.0:1787829346.690467:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb992314780 x1874672572602624/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787829346.690495:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829346.690499:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787829346.690501:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829346.690504:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 2298 00000100:00080000:3.0:1787829346.690508:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829346.690512:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:3.0:1787829346.690522:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787829346.690536:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb992315e00 x1874672572602752/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787829346.690567:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1787829346.690568:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb992314780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572602624:192.168.203.129@tcp:250:kworker.0 00000100:00080000:3.0:1787829346.690574:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1787829346.690579:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829346.690587:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 2298 00000100:00080000:3.0:1787829346.690591:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829346.690596:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:3.0:1787829346.690602:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787829346.690621:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb992316940 x1874672572602880/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1787829346.690660:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787829346.690719:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829346/real 1787829346] req@ffff8cb992314780 x1874672572602624/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829371 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829346.690743:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb992314780 x1874672572602624/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829371 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829346.690756:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829346.690760:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787829346.690775:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb992314780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572602624:192.168.203.129@tcp:250:kworker.0 00000100:00100000:0.0:1787829346.690808:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb992315e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572602752:192.168.203.129@tcp:38:kworker.0 00000800:00000100:0.0:1787829346.690834:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787829346.690845:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829346/real 1787829346] req@ffff8cb992315e00 x1874672572602752/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829371 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829346.690859:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb992315e00 x1874672572602752/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829371 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829346.690868:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829346.690881:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787829346.690886:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb992315e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572602752:192.168.203.129@tcp:38:kworker.0 00000100:00100000:0.0:1787829346.690894:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb992316940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572602880:192.168.203.129@tcp:38:kworker.0 00000800:00000100:0.0:1787829346.690912:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787829346.690921:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829346/real 1787829346] req@ffff8cb992316940 x1874672572602880/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829371 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829346.690929:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb992316940 x1874672572602880/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829371 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829346.690935:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829346.690938:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787829346.690940:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb992316940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572602880:192.168.203.129@tcp:38:kworker.0 00010000:00010000:1.0:1787829347.522405:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829348.546427:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829349.570433:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787829350.594386:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829351.618425:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787829351.810364:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829351.810378:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787829351.810384:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829351.810397:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 2303 00000100:00080000:3.0:1787829351.810411:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829351.810421:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:3.0:1787829351.810465:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb992315a40 x1874672572603008/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787829351.810504:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829351.810508:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787829351.810510:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829351.810515:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 2303 00000100:00080000:3.0:1787829351.810520:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829351.810525:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:1.0:1787829351.810551:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb992315a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572603008:192.168.203.129@tcp:250:kworker.0 00000008:00100000:3.0:1787829351.810555:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787829351.810575:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb9923170c0 x1874672572603136/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787829351.810590:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829351.810594:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1787829351.810597:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829351.810602:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 2303 00000100:00080000:3.0:1787829351.810607:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829351.810612:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:3.0:1787829351.810619:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000800:00000100:1.0:1787829351.810631:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:3.0:1787829351.810636:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb992314f00 x1874672572603264/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787829351.810677:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829351/real 1787829351] req@ffff8cb992315a40 x1874672572603008/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829376 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829351.810704:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb992315a40 x1874672572603008/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829376 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829351.810714:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829351.810718:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787829351.810722:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb992315a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572603008:192.168.203.129@tcp:250:kworker.0 00000100:00100000:1.0:1787829351.810746:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb9923170c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572603136:192.168.203.129@tcp:38:kworker.0 00000800:00000100:1.0:1787829351.810765:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787829351.810775:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829351/real 1787829351] req@ffff8cb9923170c0 x1874672572603136/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829376 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829351.810794:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb9923170c0 x1874672572603136/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829376 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829351.810803:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829351.810807:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787829351.810812:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb9923170c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572603136:192.168.203.129@tcp:38:kworker.0 00000100:00100000:1.0:1787829351.810820:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb992314f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572603264:192.168.203.129@tcp:38:kworker.0 00000800:00000100:1.0:1787829351.810843:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787829351.810856:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829351/real 1787829351] req@ffff8cb992314f00 x1874672572603264/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829376 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829351.810870:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb992314f00 x1874672572603264/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829376 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829351.810879:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829351.810883:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787829351.810887:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb992314f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572603264:192.168.203.129@tcp:38:kworker.0 00010000:00010000:2.0:1787829352.642419:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787829353.666399:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829354.690415:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829355.714430:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787829356.738422:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787829356.930413:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829356.930428:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787829356.930434:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829356.930455:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 2309 00000100:00080000:1.0:1787829356.930468:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829356.930477:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:1.0:1787829356.930544:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f3b840 x1874672572603392/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829356.930585:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829356.930589:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787829356.930591:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829356.930596:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 2309 00000100:00080000:1.0:1787829356.930601:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829356.930606:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:1.0:1787829356.930619:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787829356.930623:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f3b840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572603392:192.168.203.129@tcp:250:kworker.0 00000100:00100000:1.0:1787829356.930637:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f39e00 x1874672572603520/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829356.930649:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829356.930653:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1787829356.930655:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829356.930660:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 2309 00000100:00080000:1.0:1787829356.930665:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829356.930670:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:1.0:1787829356.930675:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787829356.930688:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f39a40 x1874672572603648/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1787829356.930702:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787829356.930770:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829356/real 1787829356] req@ffff8cb999f3b840 x1874672572603392/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829381 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829356.930786:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f3b840 x1874672572603392/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829381 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829356.930796:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829356.930799:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787829356.930802:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f3b840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572603392:192.168.203.129@tcp:250:kworker.0 00000100:00100000:0.0:1787829356.930826:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f39e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572603520:192.168.203.129@tcp:38:kworker.0 00000800:00000100:0.0:1787829356.930852:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787829356.930862:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829356/real 1787829356] req@ffff8cb999f39e00 x1874672572603520/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829381 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829356.930871:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f39e00 x1874672572603520/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829381 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829356.930877:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829356.930879:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787829356.930882:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f39e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572603520:192.168.203.129@tcp:38:kworker.0 00000100:00100000:0.0:1787829356.930886:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f39a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572603648:192.168.203.129@tcp:38:kworker.0 00000800:00000100:0.0:1787829356.930898:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787829356.930904:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829356/real 1787829356] req@ffff8cb999f39a40 x1874672572603648/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829381 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829356.930918:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f39a40 x1874672572603648/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829381 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829356.930935:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829356.930937:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787829356.930940:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f39a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572603648:192.168.203.129@tcp:38:kworker.0 00010000:00010000:2.0:1787829357.762425:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829358.786412:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787829359.810466:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829360.834466:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829361.858463:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787829362.050396:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829362.050406:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787829362.050426:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829362.050435:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 2314 00000100:00080000:1.0:1787829362.050447:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829362.050454:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:1.0:1787829362.050483:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f38b40 x1874672572603776/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829362.050505:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829362.050508:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787829362.050509:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829362.050512:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 2314 00000100:00080000:1.0:1787829362.050514:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829362.050516:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:1.0:1787829362.050524:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787829362.050532:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f3b480 x1874672572603904/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829362.050540:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829362.050542:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1787829362.050544:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829362.050546:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 2314 00000100:00080000:1.0:1787829362.050548:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829362.050554:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:0.0:1787829362.050556:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f38b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572603776:192.168.203.129@tcp:250:kworker.0 00000008:00100000:1.0:1787829362.050557:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787829362.050569:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f3b0c0 x1874672572604032/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1787829362.050615:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787829362.050653:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829362/real 1787829362] req@ffff8cb999f38b40 x1874672572603776/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829387 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829362.050668:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f38b40 x1874672572603776/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829387 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829362.050677:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829362.050680:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787829362.050683:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f38b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572603776:192.168.203.129@tcp:250:kworker.0 00000100:00100000:0.0:1787829362.050688:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f3b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572603904:192.168.203.129@tcp:38:kworker.0 00000800:00000100:0.0:1787829362.050702:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787829362.050710:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829362/real 1787829362] req@ffff8cb999f3b480 x1874672572603904/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829387 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829362.050718:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f3b480 x1874672572603904/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829387 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829362.050724:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829362.050726:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787829362.050728:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f3b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572603904:192.168.203.129@tcp:38:kworker.0 00000100:00100000:0.0:1787829362.050754:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f3b0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572604032:192.168.203.129@tcp:38:kworker.0 00000800:00000100:0.0:1787829362.050768:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787829362.050775:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829362/real 1787829362] req@ffff8cb999f3b0c0 x1874672572604032/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829387 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829362.050782:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f3b0c0 x1874672572604032/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829387 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829362.050788:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829362.050790:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787829362.050792:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f3b0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572604032:192.168.203.129@tcp:38:kworker.0 00010000:00010000:1.0:1787829362.882510:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829363.906411:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829364.930423:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787829365.954433:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829366.978475:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787829367.170356:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829367.170367:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787829367.170372:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829367.170383:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 2319 00000100:00080000:1.0:1787829367.170395:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829367.170401:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:1.0:1787829367.170448:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f38f00 x1874672572604160/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829367.170474:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829367.170477:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787829367.170479:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829367.170483:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 2319 00000100:00080000:1.0:1787829367.170487:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829367.170491:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:1.0:1787829367.170503:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787829367.170518:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f392c0 x1874672572604288/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829367.170530:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829367.170534:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1787829367.170536:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829367.170541:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 2319 00000100:00100000:0.0:1787829367.170549:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f38f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572604160:192.168.203.129@tcp:250:kworker.0 00000100:00080000:1.0:1787829367.170563:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829367.170568:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:1.0:1787829367.170573:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787829367.170585:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f3a580 x1874672572604416/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1787829367.170631:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787829367.170682:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829367/real 1787829367] req@ffff8cb999f38f00 x1874672572604160/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829392 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829367.170713:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f38f00 x1874672572604160/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829392 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829367.170727:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829367.170731:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787829367.170735:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f38f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572604160:192.168.203.129@tcp:250:kworker.0 00000100:00100000:0.0:1787829367.170744:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f392c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572604288:192.168.203.129@tcp:38:kworker.0 00000800:00000100:0.0:1787829367.170763:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787829367.170773:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829367/real 1787829367] req@ffff8cb999f392c0 x1874672572604288/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829392 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829367.170795:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f392c0 x1874672572604288/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829392 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829367.170801:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829367.170804:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787829367.170806:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f392c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572604288:192.168.203.129@tcp:38:kworker.0 00000100:00100000:0.0:1787829367.170833:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f3a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572604416:192.168.203.129@tcp:38:kworker.0 00000800:00000100:0.0:1787829367.170849:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787829367.170856:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829367/real 1787829367] req@ffff8cb999f3a580 x1874672572604416/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829392 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829367.170864:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f3a580 x1874672572604416/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829392 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829367.170870:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829367.170872:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787829367.170874:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f3a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572604416:192.168.203.129@tcp:38:kworker.0 00010000:00010000:2.0:1787829368.002444:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787829369.026476:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829370.050410:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829371.074443:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787829372.098455:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787829372.290438:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829372.290452:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787829372.290458:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829372.290471:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 2324 00000100:00080000:3.0:1787829372.290484:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829372.290493:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:3.0:1787829372.290542:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb9923161c0 x1874672572604544/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787829372.290573:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829372.290578:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787829372.290581:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829372.290585:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 2324 00000100:00080000:3.0:1787829372.290618:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829372.290624:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:3.0:1787829372.290639:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787829372.290648:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb9923161c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572604544:192.168.203.129@tcp:250:kworker.0 00000100:00100000:3.0:1787829372.290658:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb992314f00 x1874672572604672/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787829372.290670:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829372.290674:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1787829372.290676:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829372.290679:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 2324 00000100:00080000:3.0:1787829372.290684:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829372.290688:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:3.0:1787829372.290692:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787829372.290701:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb9923170c0 x1874672572604800/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1787829372.290720:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787829372.290761:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829372/real 1787829372] req@ffff8cb9923161c0 x1874672572604544/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829397 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829372.290775:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb9923161c0 x1874672572604544/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829397 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829372.290802:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829372.290805:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787829372.290808:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb9923161c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572604544:192.168.203.129@tcp:250:kworker.0 00000100:00100000:0.0:1787829372.290830:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb992314f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572604672:192.168.203.129@tcp:38:kworker.0 00000800:00000100:0.0:1787829372.290848:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787829372.290857:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829372/real 1787829372] req@ffff8cb992314f00 x1874672572604672/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829397 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829372.290865:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb992314f00 x1874672572604672/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829397 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829372.290870:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829372.290873:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787829372.290875:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb992314f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572604672:192.168.203.129@tcp:38:kworker.0 00000100:00100000:0.0:1787829372.290878:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb9923170c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572604800:192.168.203.129@tcp:38:kworker.0 00000800:00000100:0.0:1787829372.290889:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787829372.290895:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829372/real 1787829372] req@ffff8cb9923170c0 x1874672572604800/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829397 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829372.290902:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb9923170c0 x1874672572604800/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829397 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829372.290907:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829372.290909:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787829372.290911:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb9923170c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572604800:192.168.203.129@tcp:38:kworker.0 00010000:00010000:2.0:1787829373.122421:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829374.146409:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787829375.170417:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829376.194423:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829377.218423:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787829377.410404:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829377.410413:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787829377.410416:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829377.410425:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 2329 00000100:00080000:2.0:1787829377.410434:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829377.410441:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:2.0:1787829377.410480:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8ed00 x1874672572604928/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829377.410510:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829377.410514:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787829377.410516:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829377.410520:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 2329 00000100:00080000:2.0:1787829377.410525:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829377.410530:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:2.0:1787829377.410540:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787829377.410557:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8f840 x1874672572605056/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829377.410569:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829377.410572:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00100000:0.0:1787829377.410586:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8ed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572604928:192.168.203.129@tcp:250:kworker.0 00000100:00080000:2.0:1787829377.410588:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829377.410600:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 2329 00000100:00080000:2.0:1787829377.410605:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829377.410608:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:2.0:1787829377.410613:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787829377.410625:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8e940 x1874672572605184/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787829377.410721:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8f840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572605056:192.168.203.129@tcp:38:kworker.0 00000100:00100000:0.0:1787829377.410750:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8e940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572605184:192.168.203.129@tcp:38:kworker.0 00010000:00010000:1.0:1787829378.242400:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829379.266457:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829380.290624:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:1.0:1787829380.482531:0:2383:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.129:988 00000400:00000100:1.0:1787829380.482583:0:2383:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.129@tcp at host 192.168.203.129 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:1.0:1787829380.482609:0:2383:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.129@tcp 00000800:00000100:1.0:1787829380.482647:0:2383:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.129@tcp 00000800:00000100:1.0:1787829380.482661:0:2383:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.129@tcp 00000100:00100000:0.0:1787829380.482697:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829377/real 1787829380] req@ffff8cb998a8f840 x1874672572605056/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829402 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829380.482726:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8f840 x1874672572605056/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829402 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829380.482742:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829380.482749:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787829380.482757:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8f840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572605056:192.168.203.129@tcp:38:kworker.0 00000100:00100000:0.0:1787829380.482770:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829377/real 1787829380] req@ffff8cb998a8e940 x1874672572605184/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829402 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829380.482785:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8e940 x1874672572605184/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829402 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829380.482795:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829380.482802:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787829380.482809:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8e940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572605184:192.168.203.129@tcp:38:kworker.0 00000100:00100000:0.0:1787829380.482818:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829377/real 1787829380] req@ffff8cb998a8ed00 x1874672572604928/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829402 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829380.482832:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8ed00 x1874672572604928/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829402 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829380.482854:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829380.482860:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787829380.482866:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8ed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572604928:192.168.203.129@tcp:250:kworker.0 00010000:00010000:2.0:1787829381.314855:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829382.338684:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787829382.530427:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829382.530438:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787829382.530443:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829382.530454:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 2334 00000100:00080000:2.0:1787829382.530465:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829382.530473:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:2.0:1787829382.530520:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8cb40 x1874672572605312/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829382.530552:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829382.530556:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787829382.530558:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829382.530562:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 2334 00000100:00080000:2.0:1787829382.530567:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829382.530571:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:0.0:1787829382.530604:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8cb40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572605312:192.168.203.129@tcp:250:kworker.0 00000008:00100000:2.0:1787829382.530609:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787829382.530628:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8c000 x1874672572605440/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829382.530642:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829382.530646:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1787829382.530649:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829382.530654:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 2334 00000100:00080000:2.0:1787829382.530659:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829382.530664:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:2.0:1787829382.530670:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787829382.530683:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8f480 x1874672572605568/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1787829382.530696:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787829382.530773:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829382/real 1787829382] req@ffff8cb998a8cb40 x1874672572605312/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829407 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829382.530796:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8cb40 x1874672572605312/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829407 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829382.530808:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829382.530812:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787829382.530817:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8cb40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572605312:192.168.203.129@tcp:250:kworker.0 00000100:00100000:0.0:1787829382.530849:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572605440:192.168.203.129@tcp:38:kworker.0 00000800:00000100:0.0:1787829382.530876:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787829382.530891:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829382/real 1787829382] req@ffff8cb998a8c000 x1874672572605440/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829407 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829382.530905:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8c000 x1874672572605440/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829407 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829382.530914:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829382.530918:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787829382.530921:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572605440:192.168.203.129@tcp:38:kworker.0 00000100:00100000:0.0:1787829382.530938:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572605568:192.168.203.129@tcp:38:kworker.0 00000800:00000100:0.0:1787829382.530978:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787829382.530992:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829382/real 1787829382] req@ffff8cb998a8f480 x1874672572605568/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829407 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829382.531003:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8f480 x1874672572605568/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829407 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829382.531010:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829382.531014:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787829382.531018:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572605568:192.168.203.129@tcp:38:kworker.0 00010000:00010000:2.0:1787829383.362843:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829384.386892:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829385.410901:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829386.435363:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829387.458589:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787829387.650378:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829387.650386:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787829387.650390:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829387.650398:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 2339 00000100:00080000:2.0:1787829387.650407:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829387.650414:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:2.0:1787829387.650455:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8d680 x1874672572605696/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829387.650506:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829387.650510:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787829387.650512:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829387.650517:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 2339 00000100:00080000:2.0:1787829387.650521:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829387.650525:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:2.0:1787829387.650535:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787829387.650559:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8e580 x1874672572605824/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787829387.650560:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8d680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572605696:192.168.203.129@tcp:250:kworker.0 00000100:00080000:2.0:1787829387.650580:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829387.650584:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1787829387.650586:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829387.650590:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 2339 00000100:00080000:2.0:1787829387.650594:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829387.650598:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:2.0:1787829387.650604:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787829387.650618:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8cf00 x1874672572605952/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1787829387.650641:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787829387.650689:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829387/real 1787829387] req@ffff8cb998a8d680 x1874672572605696/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829412 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829387.650708:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8d680 x1874672572605696/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829412 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829387.650718:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829387.650722:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787829387.650725:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8d680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572605696:192.168.203.129@tcp:250:kworker.0 00000100:00100000:0.0:1787829387.650754:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8e580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572605824:192.168.203.129@tcp:38:kworker.0 00000800:00000100:0.0:1787829387.650781:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787829387.650797:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829387/real 1787829387] req@ffff8cb998a8e580 x1874672572605824/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829412 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829387.650827:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8e580 x1874672572605824/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829412 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829387.650835:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829387.650840:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787829387.650843:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8e580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572605824:192.168.203.129@tcp:38:kworker.0 00000100:00100000:0.0:1787829387.650851:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8cf00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572605952:192.168.203.129@tcp:38:kworker.0 00000800:00000100:0.0:1787829387.650878:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787829387.650890:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829387/real 1787829387] req@ffff8cb998a8cf00 x1874672572605952/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829412 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829387.650924:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8cf00 x1874672572605952/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829412 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829387.650935:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829387.650939:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787829387.650944:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8cf00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572605952:192.168.203.129@tcp:38:kworker.0 00010000:00010000:2.0:1787829388.483083:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829389.506735:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829390.530407:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829391.554470:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829392.578413:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787829392.770368:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829392.770379:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787829392.770383:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829392.770394:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 2344 00000100:00080000:3.0:1787829392.770405:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829392.770411:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:3.0:1787829392.770453:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb992315a40 x1874672572606080/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787829392.770484:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829392.770511:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787829392.770514:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829392.770519:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 2344 00000100:00080000:3.0:1787829392.770524:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829392.770531:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:3.0:1787829392.770543:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787829392.770547:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb992315a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572606080:192.168.203.129@tcp:250:kworker.0 00000100:00100000:3.0:1787829392.770557:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb992316940 x1874672572606208/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787829392.770570:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829392.770574:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1787829392.770576:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829392.770582:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 2344 00000100:00080000:3.0:1787829392.770587:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829392.770593:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:3.0:1787829392.770600:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787829392.770615:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb992315e00 x1874672572606336/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1787829392.770641:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787829392.770706:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829392/real 1787829392] req@ffff8cb992315a40 x1874672572606080/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829417 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829392.770729:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb992315a40 x1874672572606080/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829417 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829392.770763:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829392.770769:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787829392.770773:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb992315a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572606080:192.168.203.129@tcp:250:kworker.0 00000100:00100000:0.0:1787829392.770802:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb992316940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572606208:192.168.203.129@tcp:38:kworker.0 00000800:00000100:0.0:1787829392.770831:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787829392.770846:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829392/real 1787829392] req@ffff8cb992316940 x1874672572606208/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829417 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829392.770861:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb992316940 x1874672572606208/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829417 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829392.770871:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829392.770886:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787829392.770890:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb992316940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572606208:192.168.203.129@tcp:38:kworker.0 00000100:00100000:0.0:1787829392.770898:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb992315e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572606336:192.168.203.129@tcp:38:kworker.0 00000800:00000100:0.0:1787829392.770919:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787829392.770931:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829392/real 1787829392] req@ffff8cb992315e00 x1874672572606336/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829417 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829392.770943:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb992315e00 x1874672572606336/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829417 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829392.770951:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829392.770955:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787829392.770958:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb992315e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572606336:192.168.203.129@tcp:38:kworker.0 00010000:00010000:2.0:1787829393.602432:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829394.626431:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829395.650418:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829396.674377:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829397.698421:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787829397.890332:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829397.890340:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787829397.890343:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829397.890351:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 2350 00000100:00080000:3.0:1787829397.890360:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829397.890366:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:3.0:1787829397.890398:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb992314780 x1874672572606464/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787829397.890433:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829397.890437:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787829397.890439:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829397.890443:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 2350 00000100:00080000:3.0:1787829397.890447:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829397.890452:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:3.0:1787829397.890461:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787829397.890474:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb992314b40 x1874672572606592/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787829397.890481:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb992314780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572606464:192.168.203.129@tcp:250:kworker.0 00000100:00080000:3.0:1787829397.890485:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829397.890488:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1787829397.890489:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829397.890493:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 2350 00000100:00080000:3.0:1787829397.890496:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829397.890499:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:3.0:1787829397.890503:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787829397.890512:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb992317480 x1874672572606720/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1787829397.890540:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787829397.890585:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829397/real 1787829397] req@ffff8cb992314780 x1874672572606464/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829422 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829397.890599:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb992314780 x1874672572606464/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829422 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829397.890607:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829397.890610:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787829397.890613:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb992314780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572606464:192.168.203.129@tcp:250:kworker.0 00000100:00100000:0.0:1787829397.890635:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb992314b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572606592:192.168.203.129@tcp:38:kworker.0 00000800:00000100:0.0:1787829397.890653:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787829397.890662:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829397/real 1787829397] req@ffff8cb992314b40 x1874672572606592/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829422 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829397.890669:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb992314b40 x1874672572606592/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829422 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829397.890675:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829397.890677:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787829397.890680:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb992314b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572606592:192.168.203.129@tcp:38:kworker.0 00000100:00100000:0.0:1787829397.890683:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb992317480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572606720:192.168.203.129@tcp:38:kworker.0 00000800:00000100:0.0:1787829397.890694:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787829397.890700:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829397/real 1787829397] req@ffff8cb992317480 x1874672572606720/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829422 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829397.890715:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb992317480 x1874672572606720/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829422 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829397.890720:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829397.890722:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787829397.890724:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb992317480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572606720:192.168.203.129@tcp:38:kworker.0 00010000:00010000:2.0:1787829398.722413:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829399.746420:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829400.770400:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829401.794460:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829402.818425:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787829403.010383:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829403.010391:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787829403.010395:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829403.010403:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 2355 00000100:00080000:2.0:1787829403.010413:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829403.010419:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:2.0:1787829403.010455:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8c780 x1874672572606848/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829403.010487:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829403.010491:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787829403.010492:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829403.010495:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 2355 00000100:00080000:2.0:1787829403.010499:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829403.010501:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:2.0:1787829403.010510:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787829403.010523:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8f0c0 x1874672572606976/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787829403.010533:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8c780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572606848:192.168.203.129@tcp:250:kworker.0 00000100:00080000:2.0:1787829403.010560:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829403.010564:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1787829403.010567:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829403.010572:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 2355 00000100:00080000:2.0:1787829403.010578:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829403.010583:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:2.0:1787829403.010587:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000800:00000100:0.0:1787829403.010599:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787829403.010600:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8fc00 x1874672572607104/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787829403.010644:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829403/real 1787829403] req@ffff8cb998a8c780 x1874672572606848/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829428 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829403.010659:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8c780 x1874672572606848/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829428 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829403.010668:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829403.010672:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787829403.010675:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8c780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572606848:192.168.203.129@tcp:250:kworker.0 00000100:00100000:0.0:1787829403.010694:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8f0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572606976:192.168.203.129@tcp:38:kworker.0 00000800:00000100:0.0:1787829403.010709:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787829403.010717:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829403/real 1787829403] req@ffff8cb998a8f0c0 x1874672572606976/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829428 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829403.010726:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8f0c0 x1874672572606976/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829428 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829403.010741:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829403.010744:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787829403.010746:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8f0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572606976:192.168.203.129@tcp:38:kworker.0 00000100:00100000:0.0:1787829403.010750:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8fc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572607104:192.168.203.129@tcp:38:kworker.0 00000800:00000100:0.0:1787829403.010764:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787829403.010771:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829403/real 1787829403] req@ffff8cb998a8fc00 x1874672572607104/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829428 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829403.010778:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8fc00 x1874672572607104/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829428 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829403.010783:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829403.010785:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787829403.010787:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8fc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572607104:192.168.203.129@tcp:38:kworker.0 00010000:00010000:2.0:1787829403.842408:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829404.866412:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829405.890420:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829406.914383:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829407.938389:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787829408.130516:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829408.130528:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787829408.130533:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829408.130543:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 2360 00000100:00080000:3.0:1787829408.130572:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829408.130580:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:3.0:1787829408.130617:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb992316580 x1874672572607232/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787829408.130650:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829408.130655:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787829408.130657:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829408.130661:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 2360 00000100:00080000:3.0:1787829408.130664:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829408.130668:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:3.0:1787829408.130685:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787829408.130702:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb992317480 x1874672572607360/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787829408.130713:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829408.130718:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1787829408.130720:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829408.130725:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 2360 00000100:00080000:3.0:1787829408.130729:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829408.130734:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:3.0:1787829408.130740:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787829408.130753:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb992314b40 x1874672572607488/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787829408.130767:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb992316580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572607232:192.168.203.129@tcp:250:kworker.0 00000800:00000100:1.0:1787829408.130911:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787829408.130987:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829408/real 1787829408] req@ffff8cb992316580 x1874672572607232/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829433 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829408.131015:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb992316580 x1874672572607232/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829433 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829408.131029:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829408.131036:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787829408.131041:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb992316580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572607232:192.168.203.129@tcp:250:kworker.0 00000100:00100000:1.0:1787829408.131053:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb992317480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572607360:192.168.203.129@tcp:38:kworker.0 00000800:00000100:1.0:1787829408.131108:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787829408.131125:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829408/real 1787829408] req@ffff8cb992317480 x1874672572607360/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829433 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829408.131140:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb992317480 x1874672572607360/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829433 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829408.131149:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829408.131153:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787829408.131157:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb992317480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572607360:192.168.203.129@tcp:38:kworker.0 00000100:00100000:1.0:1787829408.131191:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb992314b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572607488:192.168.203.129@tcp:38:kworker.0 00000800:00000100:1.0:1787829408.131214:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787829408.131227:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829408/real 1787829408] req@ffff8cb992314b40 x1874672572607488/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829433 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829408.131239:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb992314b40 x1874672572607488/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829433 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829408.131249:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829408.131253:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787829408.131257:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb992314b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572607488:192.168.203.129@tcp:38:kworker.0 00010000:00010000:2.0:1787829408.962389:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829409.986409:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829411.010450:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829412.034394:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829413.058433:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787829413.250386:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829413.250398:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787829413.250402:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829413.250413:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 2365 00000100:00080000:1.0:1787829413.250423:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829413.250446:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:1.0:1787829413.250476:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f383c0 x1874672572607616/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829413.250501:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829413.250504:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787829413.250505:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829413.250508:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 2365 00000100:00080000:1.0:1787829413.250511:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829413.250513:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:1.0:1787829413.250520:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787829413.250528:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f3a580 x1874672572607744/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829413.250535:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829413.250537:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1787829413.250538:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829413.250541:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 2365 00000100:00080000:1.0:1787829413.250542:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829413.250544:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:1.0:1787829413.250547:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787829413.250555:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f392c0 x1874672572607872/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787829413.250575:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f383c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572607616:192.168.203.129@tcp:250:kworker.0 00000800:00000100:0.0:1787829413.250633:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787829413.250664:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829413/real 1787829413] req@ffff8cb999f383c0 x1874672572607616/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829438 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829413.250682:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f383c0 x1874672572607616/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829438 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829413.250693:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829413.250698:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787829413.250710:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f383c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572607616:192.168.203.129@tcp:250:kworker.0 00000100:00100000:0.0:1787829413.250718:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f3a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572607744:192.168.203.129@tcp:38:kworker.0 00000800:00000100:0.0:1787829413.250759:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787829413.250773:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829413/real 1787829413] req@ffff8cb999f3a580 x1874672572607744/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829438 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829413.250785:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f3a580 x1874672572607744/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829438 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829413.250790:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829413.250793:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787829413.250795:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f3a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572607744:192.168.203.129@tcp:38:kworker.0 00000100:00100000:0.0:1787829413.250799:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f392c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572607872:192.168.203.129@tcp:38:kworker.0 00000800:00000100:0.0:1787829413.250813:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787829413.250820:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829413/real 1787829413] req@ffff8cb999f392c0 x1874672572607872/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829438 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829413.250827:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f392c0 x1874672572607872/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829438 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829413.250833:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829413.250834:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787829413.250837:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f392c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572607872:192.168.203.129@tcp:38:kworker.0 00010000:00010000:2.0:1787829414.082399:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829415.106468:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829416.130373:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829417.154404:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829418.178365:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787829418.370344:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829418.370353:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787829418.370373:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829418.370382:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 2370 00000100:00080000:1.0:1787829418.370392:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829418.370399:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:1.0:1787829418.370429:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f38f00 x1874672572608000/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829418.370457:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829418.370460:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787829418.370461:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829418.370464:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 2370 00000100:00080000:1.0:1787829418.370466:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829418.370468:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:1.0:1787829418.370476:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787829418.370484:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f3b0c0 x1874672572608128/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829418.370492:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829418.370497:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00100000:0.0:1787829418.370498:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f38f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572608000:192.168.203.129@tcp:250:kworker.0 00000100:00080000:1.0:1787829418.370499:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829418.370506:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 2370 00000100:00080000:1.0:1787829418.370510:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829418.370512:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:1.0:1787829418.370515:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787829418.370524:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f3b480 x1874672572608256/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1787829418.370561:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787829418.370607:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829418/real 1787829418] req@ffff8cb999f38f00 x1874672572608000/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829443 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829418.370624:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f38f00 x1874672572608000/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829443 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829418.370632:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829418.370635:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787829418.370638:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f38f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572608000:192.168.203.129@tcp:250:kworker.0 00000100:00100000:0.0:1787829418.370659:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f3b0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572608128:192.168.203.129@tcp:38:kworker.0 00000800:00000100:0.0:1787829418.370676:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787829418.370685:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829418/real 1787829418] req@ffff8cb999f3b0c0 x1874672572608128/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829443 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829418.370693:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f3b0c0 x1874672572608128/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829443 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829418.370712:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829418.370715:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787829418.370717:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f3b0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572608128:192.168.203.129@tcp:38:kworker.0 00000100:00100000:0.0:1787829418.370721:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f3b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572608256:192.168.203.129@tcp:38:kworker.0 00000800:00000100:0.0:1787829418.370735:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787829418.370742:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829418/real 1787829418] req@ffff8cb999f3b480 x1874672572608256/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829443 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829418.370750:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f3b480 x1874672572608256/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829443 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829418.370766:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829418.370769:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787829418.370771:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f3b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572608256:192.168.203.129@tcp:38:kworker.0 00010000:00010000:2.0:1787829419.202417:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829420.226414:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829421.250389:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829422.274463:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829423.298474:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787829423.490486:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829423.490497:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787829423.490503:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829423.490520:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 2375 00000100:00080000:1.0:1787829423.490537:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829423.490546:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:1.0:1787829423.490598:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f38b40 x1874672572608384/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829423.490631:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829423.490637:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787829423.490640:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829423.490645:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 2375 00000100:00080000:1.0:1787829423.490651:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829423.490656:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:3.0:1787829423.490675:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f38b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572608384:192.168.203.129@tcp:250:kworker.0 00000008:00100000:1.0:1787829423.490681:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787829423.490700:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f39a40 x1874672572608512/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829423.490712:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829423.490715:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1787829423.490717:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829423.490721:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 2375 00000100:00080000:1.0:1787829423.490724:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829423.490728:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:1.0:1787829423.490738:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787829423.490749:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f39e00 x1874672572608640/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1787829423.491021:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:3.0:1787829423.491072:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829423/real 1787829423] req@ffff8cb999f38b40 x1874672572608384/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829448 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787829423.491093:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f38b40 x1874672572608384/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829448 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787829423.491105:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829423.491152:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:3.0:1787829423.491157:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f38b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572608384:192.168.203.129@tcp:250:kworker.0 00000100:00100000:3.0:1787829423.491185:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f39a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572608512:192.168.203.129@tcp:38:kworker.0 00000800:00000100:3.0:1787829423.491213:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:3.0:1787829423.491224:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829423/real 1787829423] req@ffff8cb999f39a40 x1874672572608512/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829448 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787829423.491233:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f39a40 x1874672572608512/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829448 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787829423.491239:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829423.491242:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:3.0:1787829423.491244:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f39a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572608512:192.168.203.129@tcp:38:kworker.0 00000100:00100000:3.0:1787829423.491248:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f39e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572608640:192.168.203.129@tcp:38:kworker.0 00000800:00000100:3.0:1787829423.491298:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:3.0:1787829423.491318:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829423/real 1787829423] req@ffff8cb999f39e00 x1874672572608640/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829448 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787829423.491356:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f39e00 x1874672572608640/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829448 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787829423.491370:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829423.491375:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:3.0:1787829423.491381:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f39e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572608640:192.168.203.129@tcp:38:kworker.0 00010000:00010000:2.0:1787829424.322480:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829425.346412:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829426.370397:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829427.394435:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829428.418422:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787829428.610438:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829428.610457:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787829428.610464:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829428.610481:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 2380 00000100:00080000:1.0:1787829428.610499:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829428.610509:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:1.0:1787829428.610569:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f3b840 x1874672572608768/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829428.610607:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829428.610611:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787829428.610614:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829428.610618:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 2380 00000100:00080000:1.0:1787829428.610625:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829428.610646:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:1.0:1787829428.610661:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787829428.610682:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f39e00 x1874672572608896/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829428.610695:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829428.610699:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1787829428.610703:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829428.610709:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 2380 00000100:00080000:1.0:1787829428.610732:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829428.610739:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:1.0:1787829428.610750:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787829428.610766:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f39a40 x1874672572609024/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1787829428.610934:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f3b840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572608768:192.168.203.129@tcp:250:kworker.0 00000800:00000100:3.0:1787829428.611034:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:3.0:1787829428.611099:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829428/real 1787829428] req@ffff8cb999f3b840 x1874672572608768/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829453 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787829428.611125:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f3b840 x1874672572608768/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829453 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787829428.611139:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829428.611144:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:3.0:1787829428.611150:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f3b840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572608768:192.168.203.129@tcp:250:kworker.0 00000100:00100000:3.0:1787829428.611180:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f39e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572608896:192.168.203.129@tcp:38:kworker.0 00000800:00000100:3.0:1787829428.611213:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:3.0:1787829428.611229:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829428/real 1787829428] req@ffff8cb999f39e00 x1874672572608896/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829453 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787829428.611244:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f39e00 x1874672572608896/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829453 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787829428.611331:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829428.611336:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:3.0:1787829428.611340:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f39e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572608896:192.168.203.129@tcp:38:kworker.0 00000100:00100000:3.0:1787829428.611348:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f39a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572609024:192.168.203.129@tcp:38:kworker.0 00000800:00000100:3.0:1787829428.611376:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:3.0:1787829428.611389:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829428/real 1787829428] req@ffff8cb999f39a40 x1874672572609024/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829453 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787829428.611402:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f39a40 x1874672572609024/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829453 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787829428.611412:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829428.611416:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:3.0:1787829428.611420:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f39a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572609024:192.168.203.129@tcp:38:kworker.0 00010000:00010000:2.0:1787829429.442400:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829430.466388:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829431.490414:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829432.514450:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829433.538401:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787829433.730326:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829433.730334:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787829433.730338:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829433.730346:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 2385 00000100:00080000:3.0:1787829433.730355:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829433.730361:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:3.0:1787829433.730391:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb992314780 x1874672572609152/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787829433.730419:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829433.730437:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787829433.730439:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829433.730444:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 2385 00000100:00080000:3.0:1787829433.730449:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829433.730454:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:3.0:1787829433.730465:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787829433.730477:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb992315e00 x1874672572609280/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787829433.730483:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb992314780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572609152:192.168.203.129@tcp:250:kworker.0 00000100:00080000:3.0:1787829433.730488:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829433.730493:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1787829433.730495:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829433.730499:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 2385 00000100:00080000:3.0:1787829433.730503:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829433.730507:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:3.0:1787829433.730513:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787829433.730527:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb992316940 x1874672572609408/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1787829433.730554:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787829433.730607:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829433/real 1787829433] req@ffff8cb992314780 x1874672572609152/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829458 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829433.730626:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb992314780 x1874672572609152/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829458 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829433.730637:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829433.730642:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787829433.730645:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb992314780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572609152:192.168.203.129@tcp:250:kworker.0 00000100:00100000:0.0:1787829433.730665:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb992315e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572609280:192.168.203.129@tcp:38:kworker.0 00000800:00000100:0.0:1787829433.730681:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787829433.730690:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829433/real 1787829433] req@ffff8cb992315e00 x1874672572609280/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829458 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829433.730698:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb992315e00 x1874672572609280/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829458 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829433.730704:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829433.730706:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787829433.730708:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb992315e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572609280:192.168.203.129@tcp:38:kworker.0 00000100:00100000:0.0:1787829433.730720:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb992316940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572609408:192.168.203.129@tcp:38:kworker.0 00000800:00000100:0.0:1787829433.730743:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787829433.730766:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829433/real 1787829433] req@ffff8cb992316940 x1874672572609408/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829458 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829433.730778:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb992316940 x1874672572609408/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829458 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829433.730786:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829433.730789:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787829433.730793:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb992316940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572609408:192.168.203.129@tcp:38:kworker.0 00010000:00010000:2.0:1787829434.562491:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829435.586427:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829436.610490:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829437.634403:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829438.658429:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787829438.850390:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829438.850402:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787829438.850406:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829438.850416:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 2391 00000100:00080000:1.0:1787829438.850427:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829438.850434:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:1.0:1787829438.850468:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f38b40 x1874672572609536/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829438.850500:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829438.850504:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787829438.850506:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829438.850510:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 2391 00000100:00080000:1.0:1787829438.850515:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829438.850520:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:1.0:1787829438.850530:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787829438.850545:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f38b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572609536:192.168.203.129@tcp:250:kworker.0 00000100:00100000:1.0:1787829438.850546:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f3b480 x1874672572609664/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829438.850572:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829438.850591:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1787829438.850593:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829438.850599:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 2391 00000100:00080000:1.0:1787829438.850617:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829438.850622:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000800:00000100:0.0:1787829438.850627:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000008:00100000:1.0:1787829438.850629:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787829438.850647:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f3b0c0 x1874672572609792/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787829438.850667:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829438/real 1787829438] req@ffff8cb999f38b40 x1874672572609536/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829463 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829438.850695:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f38b40 x1874672572609536/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829463 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829438.850703:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829438.850706:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787829438.850710:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f38b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572609536:192.168.203.129@tcp:250:kworker.0 00000100:00100000:0.0:1787829438.850735:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f3b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572609664:192.168.203.129@tcp:38:kworker.0 00000800:00000100:0.0:1787829438.850766:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787829438.850777:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829438/real 1787829438] req@ffff8cb999f3b480 x1874672572609664/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829463 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829438.850792:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f3b480 x1874672572609664/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829463 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829438.850798:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829438.850801:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787829438.850803:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f3b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572609664:192.168.203.129@tcp:38:kworker.0 00000100:00100000:0.0:1787829438.850807:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f3b0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572609792:192.168.203.129@tcp:38:kworker.0 00000800:00000100:0.0:1787829438.850821:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787829438.850828:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829438/real 1787829438] req@ffff8cb999f3b0c0 x1874672572609792/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829463 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829438.850836:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f3b0c0 x1874672572609792/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829463 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829438.850841:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829438.850843:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787829438.850845:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f3b0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572609792:192.168.203.129@tcp:38:kworker.0 00010000:00010000:2.0:1787829439.682445:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829440.706454:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829441.730390:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829442.754431:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829443.778441:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787829443.970384:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829443.970392:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787829443.970396:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829443.970404:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 2396 00000100:00080000:3.0:1787829443.970414:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829443.970420:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:3.0:1787829443.970449:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb992315a40 x1874672572609920/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787829443.970483:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829443.970486:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787829443.970487:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829443.970490:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 2396 00000100:00080000:3.0:1787829443.970493:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829443.970495:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:3.0:1787829443.970503:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787829443.970512:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb9923170c0 x1874672572610048/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787829443.970531:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829443.970534:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1787829443.970536:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829443.970539:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 2396 00000100:00080000:3.0:1787829443.970542:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829443.970545:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:3.0:1787829443.970550:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787829443.970556:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb992315a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572609920:192.168.203.129@tcp:250:kworker.0 00000100:00100000:3.0:1787829443.970561:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb992314f00 x1874672572610176/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787829443.970663:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb9923170c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572610048:192.168.203.129@tcp:38:kworker.0 00000100:00100000:0.0:1787829443.970706:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb992314f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572610176:192.168.203.129@tcp:38:kworker.0 00010000:00010000:2.0:1787829444.802419:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787829445.826408:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829446.850818:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:2.0:1787829447.042517:0:2384:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.129:988 00000400:00000100:2.0:1787829447.042565:0:2384:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.129@tcp at host 192.168.203.129 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:2.0:1787829447.042583:0:2384:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.129@tcp 00000800:00000100:2.0:1787829447.042630:0:2384:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.129@tcp 00000800:00000100:2.0:1787829447.042641:0:2384:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.129@tcp 00000100:00100000:0.0:1787829447.042725:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829443/real 1787829447] req@ffff8cb992314f00 x1874672572610176/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829468 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829447.042775:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb992314f00 x1874672572610176/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829468 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829447.042788:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829447.042793:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787829447.042798:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb992314f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572610176:192.168.203.129@tcp:38:kworker.0 00000100:00100000:0.0:1787829447.042806:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829443/real 1787829447] req@ffff8cb992315a40 x1874672572609920/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829468 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829447.042822:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb992315a40 x1874672572609920/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829468 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829447.042831:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829447.042835:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787829447.042839:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb992315a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572609920:192.168.203.129@tcp:250:kworker.0 00000100:00100000:0.0:1787829447.042846:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829443/real 1787829447] req@ffff8cb9923170c0 x1874672572610048/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829468 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829447.042859:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb9923170c0 x1874672572610048/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829468 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829447.042868:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829447.042872:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787829447.042876:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb9923170c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572610048:192.168.203.129@tcp:38:kworker.0 00010000:00010000:2.0:1787829447.875301:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829448.899165:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787829449.090326:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829449.090335:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787829449.090339:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829449.090348:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 2401 00000100:00080000:3.0:1787829449.090373:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829449.090382:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:3.0:1787829449.090415:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb9923161c0 x1874672572610304/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787829449.090438:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829449.090441:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787829449.090442:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829449.090448:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 2401 00000100:00080000:3.0:1787829449.090451:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829449.090455:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:3.0:1787829449.090463:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787829449.090473:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb9923170c0 x1874672572610432/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787829449.090480:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829449.090495:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1787829449.090497:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829449.090501:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 2401 00000100:00080000:3.0:1787829449.090504:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829449.090506:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:3.0:1787829449.090512:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787829449.090513:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb9923161c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572610304:192.168.203.129@tcp:250:kworker.0 00000100:00100000:3.0:1787829449.090524:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb992315a40 x1874672572610560/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1787829449.090580:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787829449.090629:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829449/real 1787829449] req@ffff8cb9923161c0 x1874672572610304/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829474 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829449.090649:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb9923161c0 x1874672572610304/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829474 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829449.090661:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829449.090665:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787829449.090669:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb9923161c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572610304:192.168.203.129@tcp:250:kworker.0 00000100:00100000:0.0:1787829449.090678:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb9923170c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572610432:192.168.203.129@tcp:38:kworker.0 00000800:00000100:0.0:1787829449.090704:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787829449.090720:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829449/real 1787829449] req@ffff8cb9923170c0 x1874672572610432/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829474 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829449.090733:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb9923170c0 x1874672572610432/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829474 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829449.090743:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829449.090746:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787829449.090749:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb9923170c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572610432:192.168.203.129@tcp:38:kworker.0 00000100:00100000:0.0:1787829449.090777:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb992315a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572610560:192.168.203.129@tcp:38:kworker.0 00000800:00000100:0.0:1787829449.090799:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787829449.090811:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829449/real 1787829449] req@ffff8cb992315a40 x1874672572610560/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829474 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829449.090840:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb992315a40 x1874672572610560/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829474 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829449.090849:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829449.090853:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787829449.090857:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb992315a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572610560:192.168.203.129@tcp:38:kworker.0 00010000:00010000:3.0:1787829449.922899:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829450.946735:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829451.971000:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787829452.994584:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829454.018971:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787829454.210354:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829454.210361:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787829454.210365:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829454.210372:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 2406 00000100:00080000:0.0:1787829454.210381:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829454.210387:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:0.0:1787829454.210418:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c6b840 x1874672572610688/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829454.210440:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829454.210442:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787829454.210461:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829454.210465:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 2406 00000100:00080000:0.0:1787829454.210468:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829454.210471:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:3.0:1787829454.210476:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c6b840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572610688:192.168.203.129@tcp:250:kworker.0 00000008:00100000:0.0:1787829454.210482:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787829454.210494:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c69a40 x1874672572610816/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829454.210523:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829454.210527:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000800:00000100:3.0:1787829454.210534:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00080000:0.0:1787829454.210539:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829454.210546:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 2406 00000100:00080000:0.0:1787829454.210551:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829454.210556:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:0.0:1787829454.210561:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787829454.210573:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c68f00 x1874672572610944/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1787829454.210580:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829454/real 1787829454] req@ffff8cb999c6b840 x1874672572610688/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829479 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787829454.210603:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c6b840 x1874672572610688/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829479 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787829454.210611:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829454.210615:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:3.0:1787829454.210617:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c6b840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572610688:192.168.203.129@tcp:250:kworker.0 00000100:00100000:3.0:1787829454.210640:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c69a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572610816:192.168.203.129@tcp:38:kworker.0 00000800:00000100:3.0:1787829454.210656:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:3.0:1787829454.210665:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829454/real 1787829454] req@ffff8cb999c69a40 x1874672572610816/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829479 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787829454.210674:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c69a40 x1874672572610816/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829479 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787829454.210679:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829454.210681:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:3.0:1787829454.210684:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c69a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572610816:192.168.203.129@tcp:38:kworker.0 00000100:00100000:3.0:1787829454.210687:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c68f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572610944:192.168.203.129@tcp:38:kworker.0 00000800:00000100:3.0:1787829454.210697:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:3.0:1787829454.210703:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829454/real 1787829454] req@ffff8cb999c68f00 x1874672572610944/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829479 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787829454.210718:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c68f00 x1874672572610944/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829479 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787829454.210724:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829454.210726:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:3.0:1787829454.210728:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c68f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572610944:192.168.203.129@tcp:38:kworker.0 00010000:00010000:2.0:1787829455.042698:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787829456.066707:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829457.090426:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829458.114441:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787829459.138404:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787829459.330396:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829459.330407:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787829459.330423:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829459.330436:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 2411 00000100:00080000:1.0:1787829459.330448:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829459.330456:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:1.0:1787829459.330506:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f38f00 x1874672572611072/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829459.330543:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829459.330547:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787829459.330549:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829459.330553:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 2411 00000100:00080000:1.0:1787829459.330557:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829459.330561:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:1.0:1787829459.330573:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787829459.330595:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f392c0 x1874672572611200/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829459.330606:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829459.330610:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1787829459.330619:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1787829459.330621:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f38f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572611072:192.168.203.129@tcp:250:kworker.0 00000100:00080000:1.0:1787829459.330630:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 2411 00000100:00080000:1.0:1787829459.330641:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829459.330647:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:1.0:1787829459.330652:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787829459.330666:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f3a580 x1874672572611328/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1787829459.330708:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787829459.330759:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829459/real 1787829459] req@ffff8cb999f38f00 x1874672572611072/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829484 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829459.330774:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f38f00 x1874672572611072/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829484 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829459.330783:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829459.330786:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787829459.330789:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f38f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572611072:192.168.203.129@tcp:250:kworker.0 00000100:00100000:2.0:1787829459.330814:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f392c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572611200:192.168.203.129@tcp:38:kworker.0 00000800:00000100:2.0:1787829459.330832:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787829459.330841:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829459/real 1787829459] req@ffff8cb999f392c0 x1874672572611200/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829484 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829459.330849:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f392c0 x1874672572611200/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829484 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829459.330855:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829459.330857:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787829459.330859:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f392c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572611200:192.168.203.129@tcp:38:kworker.0 00000100:00100000:2.0:1787829459.330880:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f3a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572611328:192.168.203.129@tcp:38:kworker.0 00000800:00000100:2.0:1787829459.330904:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787829459.330917:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829459/real 1787829459] req@ffff8cb999f3a580 x1874672572611328/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829484 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829459.330932:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f3a580 x1874672572611328/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829484 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829459.330938:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829459.330941:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787829459.330943:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f3a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572611328:192.168.203.129@tcp:38:kworker.0 00010000:00010000:2.0:1787829460.162433:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829461.186457:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787829462.210461:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829463.234446:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829464.258442:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787829464.450388:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829464.450399:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787829464.450403:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829464.450413:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 2416 00000100:00080000:3.0:1787829464.450425:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829464.450432:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:3.0:1787829464.450466:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb992314f00 x1874672572611456/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787829464.450494:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829464.450498:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787829464.450500:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829464.450505:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 2416 00000100:00080000:3.0:1787829464.450509:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829464.450513:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:3.0:1787829464.450523:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787829464.450552:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb992316940 x1874672572611584/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787829464.450566:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829464.450573:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00100000:2.0:1787829464.450575:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb992314f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572611456:192.168.203.129@tcp:250:kworker.0 00000100:00080000:3.0:1787829464.450577:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829464.450592:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 2416 00000100:00080000:3.0:1787829464.450598:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829464.450602:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:3.0:1787829464.450609:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787829464.450626:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb992315e00 x1874672572611712/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1787829464.450699:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787829464.450757:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829464/real 1787829464] req@ffff8cb992314f00 x1874672572611456/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829489 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829464.450775:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb992314f00 x1874672572611456/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829489 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829464.450786:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829464.450790:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787829464.450793:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb992314f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572611456:192.168.203.129@tcp:250:kworker.0 00000100:00100000:2.0:1787829464.450837:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb992316940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572611584:192.168.203.129@tcp:38:kworker.0 00000800:00000100:2.0:1787829464.450857:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787829464.450867:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829464/real 1787829464] req@ffff8cb992316940 x1874672572611584/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829489 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829464.450876:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb992316940 x1874672572611584/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829489 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829464.450882:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829464.450885:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787829464.450887:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb992316940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572611584:192.168.203.129@tcp:38:kworker.0 00000100:00100000:2.0:1787829464.450892:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb992315e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572611712:192.168.203.129@tcp:38:kworker.0 00000800:00000100:2.0:1787829464.450904:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787829464.450910:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829464/real 1787829464] req@ffff8cb992315e00 x1874672572611712/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829489 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829464.450918:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb992315e00 x1874672572611712/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829489 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829464.450923:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829464.450925:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787829464.450927:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb992315e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572611712:192.168.203.129@tcp:38:kworker.0 00010000:00010000:3.0:1787829465.282402:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829466.306431:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829467.330419:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787829468.354411:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829469.378430:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787829469.570410:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829469.570420:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787829469.570425:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829469.570435:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 2421 00000100:00080000:1.0:1787829469.570447:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829469.570453:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:1.0:1787829469.570496:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f383c0 x1874672572611840/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829469.570528:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829469.570533:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787829469.570535:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829469.570540:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 2421 00000100:00080000:1.0:1787829469.570544:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1787829469.570569:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f383c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572611840:192.168.203.129@tcp:250:kworker.0 00000100:00080000:1.0:1787829469.570572:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:1.0:1787829469.570584:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787829469.570600:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f3a580 x1874672572611968/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829469.570608:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829469.570611:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1787829469.570613:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829469.570616:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 2421 00000100:00080000:1.0:1787829469.570620:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829469.570622:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:1.0:1787829469.570626:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787829469.570634:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f392c0 x1874672572612096/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1787829469.570655:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787829469.570710:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829469/real 1787829469] req@ffff8cb999f383c0 x1874672572611840/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829494 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829469.570731:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f383c0 x1874672572611840/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829494 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829469.570744:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829469.570749:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787829469.570754:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f383c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572611840:192.168.203.129@tcp:250:kworker.0 00000100:00100000:2.0:1787829469.570783:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f3a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572611968:192.168.203.129@tcp:38:kworker.0 00000800:00000100:2.0:1787829469.570810:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787829469.570825:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829469/real 1787829469] req@ffff8cb999f3a580 x1874672572611968/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829494 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829469.570840:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f3a580 x1874672572611968/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829494 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829469.570849:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829469.570853:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787829469.570857:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f3a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572611968:192.168.203.129@tcp:38:kworker.0 00000100:00100000:2.0:1787829469.570865:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f392c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572612096:192.168.203.129@tcp:38:kworker.0 00000800:00000100:2.0:1787829469.570888:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787829469.570901:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829469/real 1787829469] req@ffff8cb999f392c0 x1874672572612096/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829494 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829469.570929:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f392c0 x1874672572612096/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829494 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829469.570939:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829469.570943:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787829469.570946:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f392c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572612096:192.168.203.129@tcp:38:kworker.0 00010000:00010000:2.0:1787829470.402457:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787829471.426405:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829472.450445:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829473.474504:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787829474.498379:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787829474.690398:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829474.690410:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787829474.690415:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829474.690425:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 2426 00000100:00080000:1.0:1787829474.690437:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829474.690456:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:1.0:1787829474.690495:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f38f00 x1874672572612224/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829474.690529:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829474.690534:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787829474.690551:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829474.690557:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 2426 00000100:00080000:1.0:1787829474.690561:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829474.690565:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:1.0:1787829474.690573:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787829474.690581:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f38f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572612224:192.168.203.129@tcp:250:kworker.0 00000100:00100000:1.0:1787829474.690587:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f3b0c0 x1874672572612352/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829474.690597:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829474.690600:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1787829474.690601:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829474.690604:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 2426 00000100:00080000:1.0:1787829474.690607:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829474.690612:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:1.0:1787829474.690616:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787829474.690629:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f3b480 x1874672572612480/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1787829474.690643:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787829474.690694:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829474/real 1787829474] req@ffff8cb999f38f00 x1874672572612224/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829499 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829474.690714:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f38f00 x1874672572612224/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829499 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829474.690725:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829474.690729:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787829474.690733:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f38f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572612224:192.168.203.129@tcp:250:kworker.0 00000100:00100000:2.0:1787829474.690770:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f3b0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572612352:192.168.203.129@tcp:38:kworker.0 00000800:00000100:2.0:1787829474.690795:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787829474.690810:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829474/real 1787829474] req@ffff8cb999f3b0c0 x1874672572612352/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829499 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829474.690849:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f3b0c0 x1874672572612352/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829499 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829474.690858:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829474.690862:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787829474.690865:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f3b0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572612352:192.168.203.129@tcp:38:kworker.0 00000100:00100000:2.0:1787829474.690873:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f3b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572612480:192.168.203.129@tcp:38:kworker.0 00000800:00000100:2.0:1787829474.690895:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787829474.690904:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829474/real 1787829474] req@ffff8cb999f3b480 x1874672572612480/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829499 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829474.690912:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f3b480 x1874672572612480/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829499 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829474.690917:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829474.690920:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787829474.690922:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f3b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572612480:192.168.203.129@tcp:38:kworker.0 00010000:00010000:2.0:1787829475.522502:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829476.546519:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787829477.570434:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829478.594475:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829479.618405:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787829479.810365:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829479.810375:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787829479.810379:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829479.810389:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 2431 00000100:00080000:1.0:1787829479.810400:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829479.810406:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:1.0:1787829479.810446:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f38b40 x1874672572612608/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829479.810498:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829479.810501:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787829479.810502:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829479.810505:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 2431 00000100:00080000:1.0:1787829479.810509:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829479.810512:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:1.0:1787829479.810521:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787829479.810531:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f39a40 x1874672572612736/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829479.810538:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829479.810541:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1787829479.810542:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829479.810545:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 2431 00000100:00080000:1.0:1787829479.810547:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829479.810549:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:1.0:1787829479.810552:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787829479.810560:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f39e00 x1874672572612864/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787829479.810566:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f38b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572612608:192.168.203.129@tcp:250:kworker.0 00000800:00000100:2.0:1787829479.810634:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787829479.810675:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829479/real 1787829479] req@ffff8cb999f38b40 x1874672572612608/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829504 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829479.810702:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f38b40 x1874672572612608/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829504 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829479.810711:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829479.810714:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787829479.810717:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f38b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572612608:192.168.203.129@tcp:250:kworker.0 00000100:00100000:2.0:1787829479.810723:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f39a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572612736:192.168.203.129@tcp:38:kworker.0 00000800:00000100:2.0:1787829479.810740:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787829479.810748:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829479/real 1787829479] req@ffff8cb999f39a40 x1874672572612736/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829504 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829479.810757:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f39a40 x1874672572612736/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829504 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829479.810762:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829479.810764:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787829479.810767:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f39a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572612736:192.168.203.129@tcp:38:kworker.0 00000100:00100000:2.0:1787829479.810792:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f39e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572612864:192.168.203.129@tcp:38:kworker.0 00000800:00000100:2.0:1787829479.810803:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787829479.810809:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829479/real 1787829479] req@ffff8cb999f39e00 x1874672572612864/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829504 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829479.810817:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f39e00 x1874672572612864/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829504 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829479.810831:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829479.810834:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787829479.810836:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f39e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572612864:192.168.203.129@tcp:38:kworker.0 00010000:00010000:3.0:1787829480.642392:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829481.666417:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829482.690405:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787829483.714341:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829484.738380:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787829484.930348:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829484.930358:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787829484.930362:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829484.930372:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 2437 00000100:00080000:1.0:1787829484.930398:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829484.930405:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:1.0:1787829484.930433:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f3b840 x1874672572612992/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829484.930459:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829484.930463:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787829484.930465:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829484.930469:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 2437 00000100:00080000:1.0:1787829484.930472:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829484.930476:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:1.0:1787829484.930485:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787829484.930500:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f39e00 x1874672572613120/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829484.930512:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829484.930514:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1787829484.930515:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829484.930518:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 2437 00000100:00080000:1.0:1787829484.930521:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829484.930525:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:1.0:1787829484.930529:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787829484.930539:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f39a40 x1874672572613248/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787829484.930542:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f3b840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572612992:192.168.203.129@tcp:250:kworker.0 00000800:00000100:2.0:1787829484.930629:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787829484.930693:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829484/real 1787829484] req@ffff8cb999f3b840 x1874672572612992/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829509 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829484.930715:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f3b840 x1874672572612992/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829509 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829484.930724:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829484.930729:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787829484.930734:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f3b840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572612992:192.168.203.129@tcp:250:kworker.0 00000100:00100000:2.0:1787829484.930744:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f39e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572613120:192.168.203.129@tcp:38:kworker.0 00000800:00000100:2.0:1787829484.930783:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787829484.930799:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829484/real 1787829484] req@ffff8cb999f39e00 x1874672572613120/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829509 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829484.930815:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f39e00 x1874672572613120/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829509 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829484.930839:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829484.930843:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787829484.930847:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f39e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572613120:192.168.203.129@tcp:38:kworker.0 00000100:00100000:2.0:1787829484.930883:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f39a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572613248:192.168.203.129@tcp:38:kworker.0 00000800:00000100:2.0:1787829484.930908:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787829484.930921:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829484/real 1787829484] req@ffff8cb999f39a40 x1874672572613248/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829509 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829484.930935:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f39a40 x1874672572613248/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829509 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829484.930945:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829484.930949:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787829484.930953:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f39a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572613248:192.168.203.129@tcp:38:kworker.0 00010000:00010000:1.0:1787829485.762423:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787829486.786339:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829487.810422:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787829488.834415:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787829489.858397:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787829490.050383:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829490.050393:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787829490.050399:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829490.050412:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 2442 00000100:00080000:1.0:1787829490.050425:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829490.050434:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:1.0:1787829490.050486:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f38b40 x1874672572613376/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829490.050519:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829490.050523:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787829490.050524:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829490.050528:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 2442 00000100:00080000:1.0:1787829490.050531:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829490.050534:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:1.0:1787829490.050545:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787829490.050560:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f3b480 x1874672572613504/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829490.050573:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829490.050576:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1787829490.050578:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829490.050585:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 2442 00000100:00100000:2.0:1787829490.050587:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f38b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572613376:192.168.203.129@tcp:250:kworker.0 00000100:00080000:1.0:1787829490.050590:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829490.050598:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:1.0:1787829490.050624:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787829490.050640:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f3b0c0 x1874672572613632/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1787829490.050673:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787829490.050730:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829490/real 1787829490] req@ffff8cb999f38b40 x1874672572613376/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829515 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829490.050752:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f38b40 x1874672572613376/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829515 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829490.050764:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829490.050768:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787829490.050779:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f38b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572613376:192.168.203.129@tcp:250:kworker.0 00000100:00100000:2.0:1787829490.050807:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f3b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572613504:192.168.203.129@tcp:38:kworker.0 00000800:00000100:2.0:1787829490.050830:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787829490.050842:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829490/real 1787829490] req@ffff8cb999f3b480 x1874672572613504/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829515 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829490.050854:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f3b480 x1874672572613504/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829515 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829490.050863:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829490.050867:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787829490.050871:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f3b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572613504:192.168.203.129@tcp:38:kworker.0 00000100:00100000:2.0:1787829490.050879:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f3b0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572613632:192.168.203.129@tcp:38:kworker.0 00000800:00000100:2.0:1787829490.050898:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787829490.050909:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829490/real 1787829490] req@ffff8cb999f3b0c0 x1874672572613632/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829515 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829490.050921:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f3b0c0 x1874672572613632/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829515 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829490.050930:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829490.050933:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787829490.050937:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f3b0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572613632:192.168.203.129@tcp:38:kworker.0 00010000:00010000:2.0:1787829490.882458:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787829491.906444:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787829492.930381:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829493.954428:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787829494.978446:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787829495.170397:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829495.170407:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787829495.170411:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829495.170422:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 2447 00000100:00080000:3.0:1787829495.170440:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829495.170462:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:3.0:1787829495.170498:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb992314780 x1874672572613760/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787829495.170523:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829495.170526:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787829495.170528:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829495.170531:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 2447 00000100:00080000:3.0:1787829495.170535:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829495.170538:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:3.0:1787829495.170548:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787829495.170559:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb992314b40 x1874672572613888/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787829495.170566:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb992314780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572613760:192.168.203.129@tcp:250:kworker.0 00000100:00080000:3.0:1787829495.170569:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829495.170576:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1787829495.170577:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829495.170582:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 2447 00000100:00080000:3.0:1787829495.170584:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829495.170587:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:3.0:1787829495.170591:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787829495.170600:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb992317480 x1874672572614016/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1787829495.170648:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787829495.170700:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829495/real 1787829495] req@ffff8cb992314780 x1874672572613760/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829520 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829495.170717:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb992314780 x1874672572613760/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829520 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829495.170727:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829495.170730:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787829495.170733:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb992314780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572613760:192.168.203.129@tcp:250:kworker.0 00000100:00100000:1.0:1787829495.170756:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb992314b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572613888:192.168.203.129@tcp:38:kworker.0 00000800:00000100:1.0:1787829495.170775:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787829495.170787:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829495/real 1787829495] req@ffff8cb992314b40 x1874672572613888/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829520 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829495.170796:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb992314b40 x1874672572613888/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829520 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829495.170802:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829495.170804:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787829495.170813:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb992314b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572613888:192.168.203.129@tcp:38:kworker.0 00000100:00100000:1.0:1787829495.170820:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb992317480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572614016:192.168.203.129@tcp:38:kworker.0 00000800:00000100:1.0:1787829495.170857:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787829495.170871:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829495/real 1787829495] req@ffff8cb992317480 x1874672572614016/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829520 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829495.170882:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb992317480 x1874672572614016/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829520 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829495.170889:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829495.170891:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787829495.170894:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb992317480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572614016:192.168.203.129@tcp:38:kworker.0 00010000:00010000:3.0:1787829496.002425:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829497.026393:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829498.050481:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787829499.074410:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829500.098433:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787829500.290344:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829500.290352:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787829500.290355:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829500.290363:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 2452 00000100:00080000:1.0:1787829500.290371:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829500.290376:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:1.0:1787829500.290402:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f38f00 x1874672572614144/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829500.290427:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829500.290429:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787829500.290430:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829500.290432:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 2452 00000100:00080000:1.0:1787829500.290434:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829500.290448:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:1.0:1787829500.290455:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787829500.290463:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f392c0 x1874672572614272/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829500.290471:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829500.290473:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1787829500.290476:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829500.290482:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 2452 00000100:00100000:2.0:1787829500.290482:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f38f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572614144:192.168.203.129@tcp:250:kworker.0 00000100:00080000:1.0:1787829500.290492:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829500.290496:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:1.0:1787829500.290499:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787829500.290510:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f3a580 x1874672572614400/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1787829500.290569:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787829500.290609:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829500/real 1787829500] req@ffff8cb999f38f00 x1874672572614144/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829525 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829500.290624:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f38f00 x1874672572614144/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829525 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829500.290638:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829500.290644:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787829500.290648:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f38f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572614144:192.168.203.129@tcp:250:kworker.0 00000100:00100000:2.0:1787829500.290673:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f392c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572614272:192.168.203.129@tcp:38:kworker.0 00000800:00000100:2.0:1787829500.290690:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787829500.290699:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829500/real 1787829500] req@ffff8cb999f392c0 x1874672572614272/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829525 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829500.290707:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f392c0 x1874672572614272/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829525 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829500.290713:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829500.290716:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787829500.290718:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f392c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572614272:192.168.203.129@tcp:38:kworker.0 00000100:00100000:2.0:1787829500.290722:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f3a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572614400:192.168.203.129@tcp:38:kworker.0 00000800:00000100:2.0:1787829500.290748:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787829500.290756:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829500/real 1787829500] req@ffff8cb999f3a580 x1874672572614400/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829525 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829500.290764:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f3a580 x1874672572614400/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829525 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829500.290769:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829500.290771:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787829500.290773:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f3a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572614400:192.168.203.129@tcp:38:kworker.0 00010000:00010000:2.0:1787829501.122425:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787829502.146376:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829503.170392:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829504.194428:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787829505.218369:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787829505.410409:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829505.410422:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787829505.410428:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829505.410440:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 2457 00000100:00080000:1.0:1787829505.410455:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829505.410464:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:1.0:1787829505.410520:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f383c0 x1874672572614528/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829505.410559:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829505.410563:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787829505.410584:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829505.410590:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 2457 00000100:00080000:1.0:1787829505.410595:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829505.410601:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:1.0:1787829505.410614:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787829505.410618:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f383c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572614528:192.168.203.129@tcp:250:kworker.0 00000100:00100000:1.0:1787829505.410644:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f3a580 x1874672572614656/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829505.410658:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829505.410661:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1787829505.410664:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829505.410668:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 2457 00000100:00080000:1.0:1787829505.410680:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829505.410685:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000800:00000100:2.0:1787829505.410691:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000008:00100000:1.0:1787829505.410694:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787829505.410710:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f392c0 x1874672572614784/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787829505.410749:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829505/real 1787829505] req@ffff8cb999f383c0 x1874672572614528/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829530 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829505.410772:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f383c0 x1874672572614528/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829530 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829505.410783:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829505.410786:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787829505.410789:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f383c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572614528:192.168.203.129@tcp:250:kworker.0 00000100:00100000:2.0:1787829505.410813:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f3a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572614656:192.168.203.129@tcp:38:kworker.0 00000800:00000100:2.0:1787829505.410833:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787829505.410862:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829505/real 1787829505] req@ffff8cb999f3a580 x1874672572614656/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829530 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829505.410876:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f3a580 x1874672572614656/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829530 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829505.410886:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829505.410889:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787829505.410893:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f3a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572614656:192.168.203.129@tcp:38:kworker.0 00000100:00100000:2.0:1787829505.410898:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f392c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572614784:192.168.203.129@tcp:38:kworker.0 00000800:00000100:2.0:1787829505.410915:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787829505.410924:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829505/real 1787829505] req@ffff8cb999f392c0 x1874672572614784/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829530 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829505.410932:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f392c0 x1874672572614784/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829530 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829505.410938:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829505.410940:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787829505.410942:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f392c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572614784:192.168.203.129@tcp:38:kworker.0 00010000:00010000:2.0:1787829506.242404:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829507.266426:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787829508.290429:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829509.314445:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829510.338472:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787829510.530404:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829510.530415:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787829510.530420:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829510.530430:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 2462 00000100:00080000:0.0:1787829510.530442:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829510.530448:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:0.0:1787829510.530498:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c6b480 x1874672572614912/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829510.530526:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829510.530529:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787829510.530530:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829510.530533:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 2462 00000100:00080000:0.0:1787829510.530536:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829510.530540:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:0.0:1787829510.530567:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787829510.530578:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c6bc00 x1874672572615040/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829510.530587:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829510.530589:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1787829510.530591:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1787829510.530596:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c6b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572614912:192.168.203.129@tcp:250:kworker.0 00000100:00080000:0.0:1787829510.530597:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 2462 00000100:00080000:0.0:1787829510.530601:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829510.530608:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:0.0:1787829510.530612:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787829510.530619:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c69e00 x1874672572615168/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787829510.530736:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c6bc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572615040:192.168.203.129@tcp:38:kworker.0 00000100:00100000:2.0:1787829510.530788:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c69e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572615168:192.168.203.129@tcp:38:kworker.0 00010000:00010000:3.0:1787829511.362384:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829512.386432:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829513.410721:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:1.0:1787829513.602567:0:2381:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.129:988 00000400:00000100:1.0:1787829513.602626:0:2381:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.129@tcp at host 192.168.203.129 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:1.0:1787829513.602658:0:2381:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.129@tcp 00000800:00000100:1.0:1787829513.602701:0:2381:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.129@tcp 00000800:00000100:1.0:1787829513.602709:0:2381:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.129@tcp 00000100:00100000:2.0:1787829513.602744:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829510/real 1787829513] req@ffff8cb999c6bc00 x1874672572615040/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829535 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829513.602777:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c6bc00 x1874672572615040/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829535 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829513.602790:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829513.602797:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787829513.602802:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c6bc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572615040:192.168.203.129@tcp:38:kworker.0 00000100:00100000:2.0:1787829513.602811:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829510/real 1787829513] req@ffff8cb999c69e00 x1874672572615168/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829535 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829513.602824:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c69e00 x1874672572615168/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829535 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829513.602832:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829513.602836:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787829513.602840:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c69e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572615168:192.168.203.129@tcp:38:kworker.0 00000100:00100000:2.0:1787829513.602846:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829510/real 1787829513] req@ffff8cb999c6b480 x1874672572614912/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829535 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829513.602857:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c6b480 x1874672572614912/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829535 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829513.602890:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829513.602894:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787829513.602898:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c6b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572614912:192.168.203.129@tcp:250:kworker.0 00010000:00010000:0.0:1787829514.434479:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787829515.458392:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787829515.650341:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829515.650352:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787829515.650356:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829515.650365:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 2467 00000100:00080000:0.0:1787829515.650374:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829515.650381:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:0.0:1787829515.650413:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c692c0 x1874672572615296/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829515.650439:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829515.650443:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787829515.650457:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829515.650462:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 2467 00000100:00080000:0.0:1787829515.650466:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829515.650470:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:2.0:1787829515.650480:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c692c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572615296:192.168.203.129@tcp:250:kworker.0 00000008:00100000:0.0:1787829515.650481:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787829515.650500:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c69680 x1874672572615424/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829515.650511:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829515.650515:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1787829515.650517:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829515.650521:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 2467 00000100:00080000:0.0:1787829515.650525:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829515.650528:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:0.0:1787829515.650533:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787829515.650545:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c68000 x1874672572615552/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1787829515.650556:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787829515.650598:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829515/real 1787829515] req@ffff8cb999c692c0 x1874672572615296/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829540 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829515.650615:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c692c0 x1874672572615296/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829540 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829515.650625:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829515.650629:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787829515.650633:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c692c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572615296:192.168.203.129@tcp:250:kworker.0 00000100:00100000:2.0:1787829515.650656:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c69680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572615424:192.168.203.129@tcp:38:kworker.0 00000800:00000100:2.0:1787829515.650678:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787829515.650689:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829515/real 1787829515] req@ffff8cb999c69680 x1874672572615424/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829540 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829515.650701:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c69680 x1874672572615424/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829540 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829515.650709:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829515.650712:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787829515.650715:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c69680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572615424:192.168.203.129@tcp:38:kworker.0 00000100:00100000:2.0:1787829515.650730:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c68000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572615552:192.168.203.129@tcp:38:kworker.0 00000800:00000100:2.0:1787829515.650751:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787829515.650762:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829515/real 1787829515] req@ffff8cb999c68000 x1874672572615552/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829540 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829515.650774:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c68000 x1874672572615552/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829540 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829515.650781:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829515.650784:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787829515.650787:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c68000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572615552:192.168.203.129@tcp:38:kworker.0 00010000:00010000:2.0:1787829516.482679:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829517.506452:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787829518.530420:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829519.554891:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829520.579041:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787829520.770407:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829520.770420:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787829520.770425:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829520.770458:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 2472 00000100:00080000:1.0:1787829520.770473:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829520.770481:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:1.0:1787829520.770525:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f38f00 x1874672572615680/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829520.770559:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829520.770564:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787829520.770566:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829520.770570:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 2472 00000100:00080000:1.0:1787829520.770575:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829520.770580:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:1.0:1787829520.770593:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787829520.770608:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f38f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572615680:192.168.203.129@tcp:250:kworker.0 00000100:00100000:1.0:1787829520.770609:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f3b0c0 x1874672572615808/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829520.770628:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829520.770632:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1787829520.770634:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829520.770639:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 2472 00000100:00080000:1.0:1787829520.770643:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829520.770647:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:1.0:1787829520.770653:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787829520.770666:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f3b480 x1874672572615936/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1787829520.770691:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787829520.770760:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829520/real 1787829520] req@ffff8cb999f38f00 x1874672572615680/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829545 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829520.770777:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f38f00 x1874672572615680/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829545 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829520.770786:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829520.770790:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787829520.770793:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f38f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572615680:192.168.203.129@tcp:250:kworker.0 00000100:00100000:2.0:1787829520.770826:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f3b0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572615808:192.168.203.129@tcp:38:kworker.0 00000800:00000100:2.0:1787829520.770845:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787829520.770855:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829520/real 1787829520] req@ffff8cb999f3b0c0 x1874672572615808/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829545 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829520.770864:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f3b0c0 x1874672572615808/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829545 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829520.770870:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829520.770872:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787829520.770875:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f3b0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572615808:192.168.203.129@tcp:38:kworker.0 00000100:00100000:2.0:1787829520.770878:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f3b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572615936:192.168.203.129@tcp:38:kworker.0 00000800:00000100:2.0:1787829520.770889:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787829520.770895:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829520/real 1787829520] req@ffff8cb999f3b480 x1874672572615936/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829545 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829520.770903:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f3b480 x1874672572615936/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829545 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829520.770908:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829520.770910:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787829520.770912:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f3b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572615936:192.168.203.129@tcp:38:kworker.0 00010000:00010000:3.0:1787829521.602667:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829522.626904:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829523.650428:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787829524.674475:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829525.698351:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787829525.890348:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829525.890359:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787829525.890363:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829525.890370:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 2478 00000100:00080000:1.0:1787829525.890379:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829525.890385:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:1.0:1787829525.890416:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f38b40 x1874672572616064/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829525.890432:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829525.890434:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787829525.890436:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829525.890438:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 2478 00000100:00080000:1.0:1787829525.890440:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829525.890443:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:1.0:1787829525.890450:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787829525.890460:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f38b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572616064:192.168.203.129@tcp:250:kworker.0 00000100:00100000:1.0:1787829525.890463:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f39a40 x1874672572616192/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829525.890470:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829525.890472:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1787829525.890475:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829525.890481:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 2478 00000100:00080000:1.0:1787829525.890483:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829525.890484:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:1.0:1787829525.890493:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787829525.890502:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f39e00 x1874672572616320/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1787829525.890527:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787829525.890564:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829525/real 1787829525] req@ffff8cb999f38b40 x1874672572616064/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829550 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829525.890576:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f38b40 x1874672572616064/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829550 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829525.890585:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829525.890589:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787829525.890592:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f38b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572616064:192.168.203.129@tcp:250:kworker.0 00000100:00100000:2.0:1787829525.890616:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f39a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572616192:192.168.203.129@tcp:38:kworker.0 00000800:00000100:2.0:1787829525.890636:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787829525.890645:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829525/real 1787829525] req@ffff8cb999f39a40 x1874672572616192/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829550 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829525.890653:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f39a40 x1874672572616192/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829550 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829525.890658:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829525.890660:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787829525.890662:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f39a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572616192:192.168.203.129@tcp:38:kworker.0 00000100:00100000:2.0:1787829525.890665:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f39e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572616320:192.168.203.129@tcp:38:kworker.0 00000800:00000100:2.0:1787829525.890676:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787829525.890681:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829525/real 1787829525] req@ffff8cb999f39e00 x1874672572616320/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829550 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829525.890705:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f39e00 x1874672572616320/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829550 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829525.890710:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829525.890712:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787829525.890714:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f39e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572616320:192.168.203.129@tcp:38:kworker.0 00010000:00010000:2.0:1787829526.722444:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787829527.746366:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829528.770438:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829529.794391:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787829530.818353:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787829531.010508:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829531.010518:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787829531.010524:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829531.010538:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 2483 00000100:00080000:1.0:1787829531.010552:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829531.010561:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:1.0:1787829531.010607:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f3b840 x1874672572616448/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829531.010644:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829531.010650:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787829531.010652:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829531.010657:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 2483 00000100:00080000:1.0:1787829531.010663:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829531.010670:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:1.0:1787829531.010683:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787829531.010697:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f3b840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572616448:192.168.203.129@tcp:250:kworker.0 00000100:00100000:1.0:1787829531.010702:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f39e00 x1874672572616576/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829531.010728:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829531.010732:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1787829531.010735:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829531.010741:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 2483 00000100:00080000:1.0:1787829531.010756:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829531.010762:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000800:00000100:2.0:1787829531.010767:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000008:00100000:1.0:1787829531.010772:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787829531.010790:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f39a40 x1874672572616704/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787829531.010811:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829531/real 1787829531] req@ffff8cb999f3b840 x1874672572616448/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829556 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829531.010828:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f3b840 x1874672572616448/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829556 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829531.010838:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829531.010841:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787829531.010845:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f3b840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572616448:192.168.203.129@tcp:250:kworker.0 00000100:00100000:2.0:1787829531.010869:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f39e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572616576:192.168.203.129@tcp:38:kworker.0 00000800:00000100:2.0:1787829531.010891:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787829531.010904:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829531/real 1787829531] req@ffff8cb999f39e00 x1874672572616576/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829556 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829531.010942:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f39e00 x1874672572616576/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829556 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829531.010951:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829531.010954:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787829531.010958:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f39e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572616576:192.168.203.129@tcp:38:kworker.0 00000100:00100000:2.0:1787829531.010963:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f39a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572616704:192.168.203.129@tcp:38:kworker.0 00000800:00000100:2.0:1787829531.010981:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787829531.010990:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829531/real 1787829531] req@ffff8cb999f39a40 x1874672572616704/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829556 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829531.010997:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f39a40 x1874672572616704/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829556 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829531.011003:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829531.011006:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787829531.011008:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f39a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572616704:192.168.203.129@tcp:38:kworker.0 00010000:00010000:2.0:1787829531.842445:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829532.866407:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787829533.890373:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829534.914359:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829535.938408:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787829536.130339:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829536.130347:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787829536.130350:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829536.130357:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 2488 00000100:00080000:1.0:1787829536.130365:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829536.130370:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:1.0:1787829536.130400:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f38b40 x1874672572616832/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829536.130419:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829536.130421:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787829536.130423:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829536.130425:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 2488 00000100:00080000:1.0:1787829536.130427:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829536.130429:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:1.0:1787829536.130435:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787829536.130442:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f3b480 x1874672572616960/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829536.130448:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829536.130462:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1787829536.130463:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829536.130466:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 2488 00000100:00080000:1.0:1787829536.130469:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829536.130472:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:1.0:1787829536.130476:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787829536.130483:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f3b0c0 x1874672572617088/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787829536.130487:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f38b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572616832:192.168.203.129@tcp:250:kworker.0 00000800:00000100:2.0:1787829536.130542:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787829536.130577:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829536/real 1787829536] req@ffff8cb999f38b40 x1874672572616832/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829561 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829536.130599:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f38b40 x1874672572616832/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829561 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829536.130606:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829536.130608:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787829536.130611:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f38b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572616832:192.168.203.129@tcp:250:kworker.0 00000100:00100000:2.0:1787829536.130616:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f3b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572616960:192.168.203.129@tcp:38:kworker.0 00000800:00000100:2.0:1787829536.130630:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787829536.130637:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829536/real 1787829536] req@ffff8cb999f3b480 x1874672572616960/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829561 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829536.130643:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f3b480 x1874672572616960/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829561 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829536.130648:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829536.130650:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787829536.130651:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f3b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572616960:192.168.203.129@tcp:38:kworker.0 00000100:00100000:2.0:1787829536.130675:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f3b0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572617088:192.168.203.129@tcp:38:kworker.0 00000800:00000100:2.0:1787829536.130686:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787829536.130692:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829536/real 1787829536] req@ffff8cb999f3b0c0 x1874672572617088/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829561 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829536.130698:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f3b0c0 x1874672572617088/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829561 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829536.130710:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829536.130712:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787829536.130714:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f3b0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572617088:192.168.203.129@tcp:38:kworker.0 00010000:00010000:3.0:1787829536.962352:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829537.986408:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829539.010398:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787829540.034369:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787829541.058404:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787829541.250341:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829541.250349:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787829541.250352:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829541.250360:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 2493 00000100:00080000:2.0:1787829541.250368:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829541.250374:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:2.0:1787829541.250401:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8e1c0 x1874672572617216/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829541.250422:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829541.250425:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787829541.250427:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829541.250430:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 2493 00000100:00080000:2.0:1787829541.250434:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829541.250438:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:2.0:1787829541.250448:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787829541.250456:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8e1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572617216:192.168.203.129@tcp:250:kworker.0 00000100:00100000:2.0:1787829541.250462:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8da40 x1874672572617344/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829541.250470:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829541.250482:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1787829541.250484:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829541.250487:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 2493 00000100:00080000:2.0:1787829541.250490:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829541.250493:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:2.0:1787829541.250497:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787829541.250505:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8de00 x1874672572617472/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1787829541.250522:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787829541.250554:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829541/real 1787829541] req@ffff8cb998a8e1c0 x1874672572617216/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829566 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829541.250569:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8e1c0 x1874672572617216/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829566 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829541.250579:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829541.250588:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787829541.250592:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8e1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572617216:192.168.203.129@tcp:250:kworker.0 00000100:00100000:0.0:1787829541.250615:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8da40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572617344:192.168.203.129@tcp:38:kworker.0 00000800:00000100:0.0:1787829541.250632:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787829541.250640:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829541/real 1787829541] req@ffff8cb998a8da40 x1874672572617344/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829566 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829541.250647:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8da40 x1874672572617344/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829566 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829541.250653:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829541.250654:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787829541.250657:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8da40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572617344:192.168.203.129@tcp:38:kworker.0 00000100:00100000:0.0:1787829541.250660:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8de00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572617472:192.168.203.129@tcp:38:kworker.0 00000800:00000100:0.0:1787829541.250670:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787829541.250675:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829541/real 1787829541] req@ffff8cb998a8de00 x1874672572617472/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829566 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829541.250682:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8de00 x1874672572617472/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829566 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829541.250686:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829541.250688:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787829541.250689:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8de00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572617472:192.168.203.129@tcp:38:kworker.0 00010000:00010000:2.0:1787829542.082410:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787829543.106361:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787829544.130420:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829545.154398:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787829546.178411:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787829546.370356:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829546.370369:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787829546.370376:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829546.370389:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 2498 00000100:00080000:2.0:1787829546.370404:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829546.370413:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:2.0:1787829546.370467:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8d2c0 x1874672572617600/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829546.370505:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829546.370509:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787829546.370512:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829546.370516:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 2498 00000100:00080000:2.0:1787829546.370520:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829546.370525:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:2.0:1787829546.370556:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787829546.370568:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8c3c0 x1874672572617728/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829546.370577:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829546.370580:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1787829546.370582:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829546.370585:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 2498 00000100:00080000:2.0:1787829546.370588:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829546.370593:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:0.0:1787829546.370596:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8d2c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572617600:192.168.203.129@tcp:250:kworker.0 00000008:00100000:2.0:1787829546.370598:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787829546.370607:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8de00 x1874672572617856/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1787829546.370678:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787829546.370729:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829546/real 1787829546] req@ffff8cb998a8d2c0 x1874672572617600/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829571 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829546.370745:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8d2c0 x1874672572617600/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829571 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829546.370755:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829546.370759:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787829546.370764:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8d2c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572617600:192.168.203.129@tcp:250:kworker.0 00000100:00100000:0.0:1787829546.370769:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8c3c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572617728:192.168.203.129@tcp:38:kworker.0 00000800:00000100:0.0:1787829546.370784:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787829546.370793:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829546/real 1787829546] req@ffff8cb998a8c3c0 x1874672572617728/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829571 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829546.370801:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8c3c0 x1874672572617728/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829571 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829546.370818:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829546.370821:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787829546.370823:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8c3c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572617728:192.168.203.129@tcp:38:kworker.0 00000100:00100000:0.0:1787829546.370849:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8de00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572617856:192.168.203.129@tcp:38:kworker.0 00000800:00000100:0.0:1787829546.370861:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787829546.370868:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829546/real 1787829546] req@ffff8cb998a8de00 x1874672572617856/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829571 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829546.370887:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8de00 x1874672572617856/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829571 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829546.370893:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829546.370895:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787829546.370897:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8de00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572617856:192.168.203.129@tcp:38:kworker.0 00010000:00010000:1.0:1787829547.202406:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829548.226460:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787829549.250359:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787829550.274405:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829551.298404:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787829551.490376:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829551.490386:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787829551.490391:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829551.490401:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 2503 00000100:00080000:2.0:1787829551.490412:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829551.490419:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:2.0:1787829551.490456:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8da40 x1874672572617984/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829551.490490:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829551.490494:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787829551.490510:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829551.490516:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 2503 00000100:00080000:2.0:1787829551.490521:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829551.490526:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:2.0:1787829551.490537:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787829551.490543:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8da40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572617984:192.168.203.129@tcp:250:kworker.0 00000100:00100000:2.0:1787829551.490556:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8e1c0 x1874672572618112/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829551.490570:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829551.490573:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1787829551.490576:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829551.490581:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 2503 00000100:00080000:2.0:1787829551.490586:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829551.490590:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:2.0:1787829551.490595:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000800:00000100:0.0:1787829551.490607:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787829551.490610:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8fc00 x1874672572618240/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787829551.490647:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829551/real 1787829551] req@ffff8cb998a8da40 x1874672572617984/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829576 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829551.490662:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8da40 x1874672572617984/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829576 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829551.490679:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829551.490684:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787829551.490688:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8da40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572617984:192.168.203.129@tcp:250:kworker.0 00000100:00100000:0.0:1787829551.490707:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8e1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572618112:192.168.203.129@tcp:38:kworker.0 00000800:00000100:0.0:1787829551.490723:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787829551.490732:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829551/real 1787829551] req@ffff8cb998a8e1c0 x1874672572618112/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829576 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829551.490740:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8e1c0 x1874672572618112/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829576 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829551.490755:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829551.490758:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787829551.490760:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8e1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572618112:192.168.203.129@tcp:38:kworker.0 00000100:00100000:0.0:1787829551.490764:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8fc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572618240:192.168.203.129@tcp:38:kworker.0 00000800:00000100:0.0:1787829551.490796:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787829551.490805:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829551/real 1787829551] req@ffff8cb998a8fc00 x1874672572618240/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829576 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829551.490813:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8fc00 x1874672572618240/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829576 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829551.490818:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829551.490821:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787829551.490823:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8fc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572618240:192.168.203.129@tcp:38:kworker.0 00010000:00010000:3.0:1787829552.322353:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787829553.346399:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829554.370364:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787829555.394423:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787829556.418488:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787829556.610396:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829556.610407:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787829556.610411:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829556.610422:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 2508 00000100:00080000:1.0:1787829556.610433:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829556.610441:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:1.0:1787829556.610485:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f38f00 x1874672572618368/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829556.610518:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829556.610522:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787829556.610524:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829556.610544:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 2508 00000100:00080000:1.0:1787829556.610550:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829556.610556:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:1.0:1787829556.610568:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787829556.610573:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f38f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572618368:192.168.203.129@tcp:250:kworker.0 00000100:00100000:1.0:1787829556.610588:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f392c0 x1874672572618496/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829556.610604:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829556.610608:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1787829556.610611:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829556.610617:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 2508 00000100:00080000:1.0:1787829556.610643:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000800:00000100:0.0:1787829556.610649:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00080000:1.0:1787829556.610651:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:1.0:1787829556.610660:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787829556.610683:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f3a580 x1874672572618624/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787829556.610704:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829556/real 1787829556] req@ffff8cb999f38f00 x1874672572618368/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829581 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829556.610730:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f38f00 x1874672572618368/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829581 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829556.610756:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829556.610760:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787829556.610774:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f38f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572618368:192.168.203.129@tcp:250:kworker.0 00000100:00100000:0.0:1787829556.610802:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f392c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572618496:192.168.203.129@tcp:38:kworker.0 00000800:00000100:0.0:1787829556.610823:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787829556.610833:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829556/real 1787829556] req@ffff8cb999f392c0 x1874672572618496/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829581 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829556.610842:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f392c0 x1874672572618496/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829581 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829556.610848:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829556.610851:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787829556.610853:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f392c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572618496:192.168.203.129@tcp:38:kworker.0 00000100:00100000:0.0:1787829556.610857:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f3a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572618624:192.168.203.129@tcp:38:kworker.0 00000800:00000100:0.0:1787829556.610869:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787829556.610875:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829556/real 1787829556] req@ffff8cb999f3a580 x1874672572618624/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829581 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829556.610887:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f3a580 x1874672572618624/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829581 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829556.610893:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829556.610895:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787829556.610898:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f3a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572618624:192.168.203.129@tcp:38:kworker.0 00010000:00010000:2.0:1787829557.442419:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787829558.466415:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787829559.490420:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829560.514462:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787829561.538400:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787829561.730300:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829561.730308:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787829561.730311:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829561.730319:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 2513 00000100:00080000:1.0:1787829561.730327:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829561.730333:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:1.0:1787829561.730362:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f383c0 x1874672572618752/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829561.730398:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829561.730401:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787829561.730402:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829561.730405:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 2513 00000100:00080000:1.0:1787829561.730408:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829561.730411:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:1.0:1787829561.730420:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787829561.730431:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f3a580 x1874672572618880/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829561.730438:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829561.730442:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1787829561.730443:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829561.730445:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 2513 00000100:00080000:1.0:1787829561.730447:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829561.730449:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:1.0:1787829561.730452:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787829561.730460:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f392c0 x1874672572619008/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787829561.730468:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f383c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572618752:192.168.203.129@tcp:250:kworker.0 00000800:00000100:0.0:1787829561.730554:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787829561.730612:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829561/real 1787829561] req@ffff8cb999f383c0 x1874672572618752/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829586 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829561.730628:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f383c0 x1874672572618752/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829586 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829561.730638:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829561.730642:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787829561.730645:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f383c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572618752:192.168.203.129@tcp:250:kworker.0 00000100:00100000:0.0:1787829561.730651:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f3a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572618880:192.168.203.129@tcp:38:kworker.0 00000800:00000100:0.0:1787829561.730669:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787829561.730677:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829561/real 1787829561] req@ffff8cb999f3a580 x1874672572618880/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829586 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829561.730686:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f3a580 x1874672572618880/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829586 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829561.730691:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829561.730694:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787829561.730696:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f3a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572618880:192.168.203.129@tcp:38:kworker.0 00000100:00100000:0.0:1787829561.730740:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f392c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572619008:192.168.203.129@tcp:38:kworker.0 00000800:00000100:0.0:1787829561.730755:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787829561.730763:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829561/real 1787829561] req@ffff8cb999f392c0 x1874672572619008/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829586 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829561.730771:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f392c0 x1874672572619008/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829586 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829561.730786:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829561.730789:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787829561.730791:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f392c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572619008:192.168.203.129@tcp:38:kworker.0 00010000:00010000:0.0:1787829562.562457:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829563.586447:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787829564.610339:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787829565.634433:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829566.658443:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787829566.850433:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829566.850443:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787829566.850449:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829566.850459:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 2519 00000100:00080000:1.0:1787829566.850484:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829566.850492:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:1.0:1787829566.850531:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f38f00 x1874672572619136/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829566.850559:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829566.850562:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787829566.850564:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829566.850567:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 2519 00000100:00080000:1.0:1787829566.850569:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829566.850572:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:1.0:1787829566.850583:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787829566.850595:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f3b0c0 x1874672572619264/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787829566.850602:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f38f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572619136:192.168.203.129@tcp:250:kworker.0 00000100:00080000:1.0:1787829566.850609:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829566.850612:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1787829566.850614:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829566.850618:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 2519 00000100:00080000:1.0:1787829566.850621:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829566.850624:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:1.0:1787829566.850628:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787829566.850637:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f3b480 x1874672572619392/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1787829566.850690:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787829566.850736:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829566/real 1787829566] req@ffff8cb999f38f00 x1874672572619136/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829591 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829566.850765:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f38f00 x1874672572619136/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829591 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829566.850775:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829566.850779:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787829566.850782:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f38f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572619136:192.168.203.129@tcp:250:kworker.0 00000100:00100000:2.0:1787829566.850808:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f3b0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572619264:192.168.203.129@tcp:38:kworker.0 00000800:00000100:2.0:1787829566.850828:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787829566.850837:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829566/real 1787829566] req@ffff8cb999f3b0c0 x1874672572619264/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829591 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829566.850845:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f3b0c0 x1874672572619264/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829591 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829566.850860:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829566.850863:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787829566.850865:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f3b0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572619264:192.168.203.129@tcp:38:kworker.0 00000100:00100000:2.0:1787829566.850871:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f3b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572619392:192.168.203.129@tcp:38:kworker.0 00000800:00000100:2.0:1787829566.850887:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787829566.850895:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829566/real 1787829566] req@ffff8cb999f3b480 x1874672572619392/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829591 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829566.850903:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f3b480 x1874672572619392/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829591 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829566.850918:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829566.850922:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787829566.850925:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f3b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572619392:192.168.203.129@tcp:38:kworker.0 00010000:00010000:3.0:1787829567.682371:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787829568.706401:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829569.730400:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787829570.754398:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829571.778393:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787829571.970365:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829571.970375:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787829571.970379:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829571.970389:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 2524 00000100:00080000:2.0:1787829571.970401:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829571.970410:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:2.0:1787829571.970446:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8f0c0 x1874672572619520/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829571.970468:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829571.970471:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787829571.970473:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829571.970475:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 2524 00000100:00080000:2.0:1787829571.970479:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829571.970483:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:2.0:1787829571.970491:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787829571.970501:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8f0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572619520:192.168.203.129@tcp:250:kworker.0 00000100:00100000:2.0:1787829571.970502:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8c780 x1874672572619648/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829571.970515:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829571.970531:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1787829571.970533:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829571.970537:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 2524 00000100:00080000:2.0:1787829571.970540:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829571.970544:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:2.0:1787829571.970547:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787829571.970557:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8cf00 x1874672572619776/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1787829571.970566:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787829571.970606:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829571/real 1787829571] req@ffff8cb998a8f0c0 x1874672572619520/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829596 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829571.970621:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8f0c0 x1874672572619520/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829596 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829571.970646:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829571.970651:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787829571.970654:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8f0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572619520:192.168.203.129@tcp:250:kworker.0 00000100:00100000:0.0:1787829571.970676:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8c780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572619648:192.168.203.129@tcp:38:kworker.0 00000800:00000100:0.0:1787829571.970701:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787829571.970715:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829571/real 1787829571] req@ffff8cb998a8c780 x1874672572619648/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829596 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829571.970728:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8c780 x1874672572619648/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829596 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829571.970748:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829571.970752:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787829571.970755:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8c780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572619648:192.168.203.129@tcp:38:kworker.0 00000100:00100000:0.0:1787829571.970762:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8cf00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572619776:192.168.203.129@tcp:38:kworker.0 00000800:00000100:0.0:1787829571.970787:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787829571.970801:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829571/real 1787829571] req@ffff8cb998a8cf00 x1874672572619776/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829596 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829571.970814:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8cf00 x1874672572619776/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829596 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829571.970823:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829571.970826:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787829571.970830:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8cf00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572619776:192.168.203.129@tcp:38:kworker.0 00010000:00010000:2.0:1787829572.802431:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787829573.826356:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829574.850433:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829575.874468:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787829576.898418:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787829577.090501:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829577.090516:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787829577.090522:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829577.090534:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 2529 00000100:00080000:0.0:1787829577.090549:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829577.090557:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:0.0:1787829577.090621:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c6a1c0 x1874672572619904/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829577.090660:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829577.090684:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787829577.090686:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829577.090692:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 2529 00000100:00080000:0.0:1787829577.090697:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829577.090703:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:0.0:1787829577.090717:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787829577.090717:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c6a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572619904:192.168.203.129@tcp:250:kworker.0 00000100:00100000:0.0:1787829577.090745:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c6a580 x1874672572620032/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829577.090762:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829577.090767:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1787829577.090769:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829577.090774:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 2529 00000100:00080000:0.0:1787829577.090781:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829577.090787:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:0.0:1787829577.090795:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787829577.090813:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c68000 x1874672572620160/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787829577.090860:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c6a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572620032:192.168.203.129@tcp:38:kworker.0 00000100:00100000:2.0:1787829577.090897:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c68000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572620160:192.168.203.129@tcp:38:kworker.0 00010000:00010000:2.0:1787829577.922440:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829578.946492:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787829579.970452:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:1.0:1787829580.162543:0:2382:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.129:988 00000400:00000100:1.0:1787829580.162638:0:2382:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.129@tcp at host 192.168.203.129 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:1.0:1787829580.162660:0:2382:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.129@tcp 00000800:00000100:1.0:1787829580.162721:0:2382:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.129@tcp 00000800:00000100:1.0:1787829580.162733:0:2382:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.129@tcp 00000100:00100000:2.0:1787829580.162814:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829577/real 1787829580] req@ffff8cb999c6a580 x1874672572620032/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829602 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829580.162845:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c6a580 x1874672572620032/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829602 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829580.162860:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829580.162866:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787829580.162871:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c6a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572620032:192.168.203.129@tcp:38:kworker.0 00000100:00100000:2.0:1787829580.162891:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829577/real 1787829580] req@ffff8cb999c68000 x1874672572620160/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829602 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829580.162906:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c68000 x1874672572620160/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829602 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829580.162913:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829580.162917:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787829580.162921:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c68000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572620160:192.168.203.129@tcp:38:kworker.0 00000100:00100000:2.0:1787829580.162929:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829577/real 1787829580] req@ffff8cb999c6a1c0 x1874672572619904/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829602 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829580.162943:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c6a1c0 x1874672572619904/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829602 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829580.162952:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829580.162956:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787829580.162959:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c6a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572619904:192.168.203.129@tcp:250:kworker.0 00010000:00010000:2.0:1787829580.994895:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829582.018572:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787829582.210395:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829582.210406:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787829582.210411:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829582.210421:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 2534 00000100:00080000:0.0:1787829582.210431:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829582.210438:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:0.0:1787829582.210472:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c69680 x1874672572620288/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829582.210512:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829582.210516:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787829582.210519:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829582.210523:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 2534 00000100:00080000:0.0:1787829582.210528:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829582.210533:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:0.0:1787829582.210544:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787829582.210577:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c692c0 x1874672572620416/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829582.210589:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829582.210593:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1787829582.210595:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829582.210599:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 2534 00000100:00080000:0.0:1787829582.210603:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829582.210607:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:0.0:1787829582.210613:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787829582.210629:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c6b480 x1874672572620544/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787829582.210631:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c69680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572620288:192.168.203.129@tcp:250:kworker.0 00000800:00000100:2.0:1787829582.210750:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787829582.210821:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829582/real 1787829582] req@ffff8cb999c69680 x1874672572620288/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829607 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829582.210847:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c69680 x1874672572620288/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829607 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829582.210883:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829582.210894:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787829582.210900:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c69680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572620288:192.168.203.129@tcp:250:kworker.0 00000100:00100000:2.0:1787829582.210911:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c692c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572620416:192.168.203.129@tcp:38:kworker.0 00000800:00000100:2.0:1787829582.210941:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787829582.210957:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829582/real 1787829582] req@ffff8cb999c692c0 x1874672572620416/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829607 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829582.210969:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c692c0 x1874672572620416/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829607 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829582.210978:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829582.210982:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787829582.210986:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c692c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572620416:192.168.203.129@tcp:38:kworker.0 00000100:00100000:2.0:1787829582.211024:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c6b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572620544:192.168.203.129@tcp:38:kworker.0 00000800:00000100:2.0:1787829582.211045:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787829582.211056:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829582/real 1787829582] req@ffff8cb999c6b480 x1874672572620544/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829607 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829582.211068:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c6b480 x1874672572620544/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829607 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829582.211077:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829582.211080:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787829582.211084:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c6b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572620544:192.168.203.129@tcp:38:kworker.0 00010000:00010000:3.0:1787829583.042385:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787829584.066816:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829585.091055:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829586.114490:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829587.138950:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787829587.330348:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829587.330357:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787829587.330360:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829587.330369:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 2539 00000100:00080000:0.0:1787829587.330395:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829587.330402:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:0.0:1787829587.330431:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c69e00 x1874672572620672/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829587.330454:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829587.330457:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787829587.330458:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829587.330461:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 2539 00000100:00080000:0.0:1787829587.330463:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829587.330465:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:0.0:1787829587.330473:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787829587.330482:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c6bc00 x1874672572620800/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829587.330490:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1787829587.330493:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c69e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572620672:192.168.203.129@tcp:250:kworker.0 00000100:00080000:0.0:1787829587.330494:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1787829587.330498:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829587.330505:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 2539 00000100:00080000:0.0:1787829587.330507:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829587.330509:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:0.0:1787829587.330512:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787829587.330521:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c68f00 x1874672572620928/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1787829587.330569:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787829587.330603:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829587/real 1787829587] req@ffff8cb999c69e00 x1874672572620672/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829612 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829587.330616:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c69e00 x1874672572620672/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829612 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829587.330624:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829587.330627:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787829587.330629:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c69e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572620672:192.168.203.129@tcp:250:kworker.0 00000100:00100000:2.0:1787829587.330648:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c6bc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572620800:192.168.203.129@tcp:38:kworker.0 00000800:00000100:2.0:1787829587.330663:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787829587.330673:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829587/real 1787829587] req@ffff8cb999c6bc00 x1874672572620800/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829612 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829587.330687:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c6bc00 x1874672572620800/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829612 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829587.330696:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829587.330700:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787829587.330703:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c6bc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572620800:192.168.203.129@tcp:38:kworker.0 00000100:00100000:2.0:1787829587.330710:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c68f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572620928:192.168.203.129@tcp:38:kworker.0 00000800:00000100:2.0:1787829587.330741:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787829587.330753:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829587/real 1787829587] req@ffff8cb999c68f00 x1874672572620928/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829612 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829587.330762:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c68f00 x1874672572620928/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829612 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829587.330767:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829587.330770:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787829587.330772:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c68f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572620928:192.168.203.129@tcp:38:kworker.0 00010000:00010000:2.0:1787829588.163252:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829589.186422:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829590.210473:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829591.234416:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829592.258451:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787829592.450456:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829592.450480:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787829592.450484:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829592.450502:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 2544 00000100:00080000:0.0:1787829592.450513:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829592.450521:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:0.0:1787829592.450554:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c69a40 x1874672572621056/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829592.450584:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829592.450588:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787829592.450591:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829592.450596:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 2544 00000100:00080000:0.0:1787829592.450600:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829592.450604:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:0.0:1787829592.450615:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787829592.450629:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c6b840 x1874672572621184/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829592.450640:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829592.450643:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1787829592.450645:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829592.450654:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 2544 00000100:00100000:2.0:1787829592.450657:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c69a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572621056:192.168.203.129@tcp:250:kworker.0 00000100:00080000:0.0:1787829592.450659:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829592.450678:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:0.0:1787829592.450687:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787829592.450702:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c68f00 x1874672572621312/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1787829592.450771:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787829592.450819:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829592/real 1787829592] req@ffff8cb999c69a40 x1874672572621056/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829617 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829592.450835:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c69a40 x1874672572621056/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829617 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829592.450845:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829592.450849:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787829592.450852:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c69a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572621056:192.168.203.129@tcp:250:kworker.0 00000100:00100000:2.0:1787829592.450880:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c6b840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572621184:192.168.203.129@tcp:38:kworker.0 00000800:00000100:2.0:1787829592.450918:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787829592.450931:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829592/real 1787829592] req@ffff8cb999c6b840 x1874672572621184/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829617 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829592.450941:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c6b840 x1874672572621184/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829617 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829592.450947:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829592.450951:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787829592.450954:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c6b840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572621184:192.168.203.129@tcp:38:kworker.0 00000100:00100000:2.0:1787829592.450962:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c68f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572621312:192.168.203.129@tcp:38:kworker.0 00000800:00000100:2.0:1787829592.450997:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787829592.451009:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829592/real 1787829592] req@ffff8cb999c68f00 x1874672572621312/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829617 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829592.451021:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c68f00 x1874672572621312/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829617 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829592.451027:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829592.451030:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787829592.451032:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c68f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572621312:192.168.203.129@tcp:38:kworker.0 00010000:00010000:2.0:1787829593.282441:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829594.306439:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829595.330398:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829596.354455:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829597.378393:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787829597.570371:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829597.570382:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787829597.570386:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829597.570397:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 2549 00000100:00080000:1.0:1787829597.570410:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829597.570418:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:1.0:1787829597.570473:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f38b40 x1874672572621440/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829597.570498:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829597.570502:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787829597.570504:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829597.570508:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 2549 00000100:00080000:1.0:1787829597.570511:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829597.570514:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:1.0:1787829597.570524:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787829597.570538:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f39a40 x1874672572621568/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787829597.570540:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f38b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572621440:192.168.203.129@tcp:250:kworker.0 00000100:00080000:1.0:1787829597.570549:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829597.570552:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1787829597.570554:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829597.570568:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 2549 00000100:00080000:1.0:1787829597.570573:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829597.570577:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:1.0:1787829597.570583:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787829597.570597:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f39e00 x1874672572621696/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1787829597.570632:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787829597.570675:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829597/real 1787829597] req@ffff8cb999f38b40 x1874672572621440/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829622 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829597.570692:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f38b40 x1874672572621440/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829622 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829597.570702:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829597.570705:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787829597.570709:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f38b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572621440:192.168.203.129@tcp:250:kworker.0 00000100:00100000:2.0:1787829597.570731:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f39a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572621568:192.168.203.129@tcp:38:kworker.0 00000800:00000100:2.0:1787829597.570754:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787829597.570767:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829597/real 1787829597] req@ffff8cb999f39a40 x1874672572621568/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829622 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829597.570779:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f39a40 x1874672572621568/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829622 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829597.570786:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829597.570790:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787829597.570793:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f39a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572621568:192.168.203.129@tcp:38:kworker.0 00000100:00100000:2.0:1787829597.570799:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f39e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572621696:192.168.203.129@tcp:38:kworker.0 00000800:00000100:2.0:1787829597.570826:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787829597.570838:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829597/real 1787829597] req@ffff8cb999f39e00 x1874672572621696/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829622 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829597.570865:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f39e00 x1874672572621696/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829622 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829597.570872:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829597.570876:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787829597.570889:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f39e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572621696:192.168.203.129@tcp:38:kworker.0 00010000:00010000:2.0:1787829598.402479:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829599.426421:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829600.450412:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829601.474434:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829602.498387:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787829602.690455:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829602.690464:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787829602.690467:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829602.690476:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 2554 00000100:00080000:1.0:1787829602.690487:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829602.690495:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:1.0:1787829602.690526:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f3b840 x1874672572621824/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829602.690541:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829602.690545:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787829602.690547:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829602.690551:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 2554 00000100:00080000:1.0:1787829602.690555:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829602.690559:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:2.0:1787829602.690566:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f3b840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572621824:192.168.203.129@tcp:250:kworker.0 00000008:00100000:1.0:1787829602.690569:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787829602.690586:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f39e00 x1874672572621952/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829602.690597:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829602.690607:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1787829602.690609:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829602.690613:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 2554 00000800:00000100:2.0:1787829602.690632:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00080000:1.0:1787829602.690778:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829602.690785:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:1.0:1787829602.690791:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787829602.690807:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f39a40 x1874672572622080/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787829602.690811:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829602/real 1787829602] req@ffff8cb999f3b840 x1874672572621824/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829627 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829602.690832:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f3b840 x1874672572621824/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829627 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829602.690844:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829602.690849:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787829602.690855:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f3b840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572621824:192.168.203.129@tcp:250:kworker.0 00000100:00100000:2.0:1787829602.690881:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f39e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572621952:192.168.203.129@tcp:38:kworker.0 00000800:00000100:2.0:1787829602.690904:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787829602.690916:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829602/real 1787829602] req@ffff8cb999f39e00 x1874672572621952/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829627 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829602.690934:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f39e00 x1874672572621952/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829627 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829602.690942:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829602.690945:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787829602.690948:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f39e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572621952:192.168.203.129@tcp:38:kworker.0 00000100:00100000:2.0:1787829602.690954:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f39a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572622080:192.168.203.129@tcp:38:kworker.0 00000800:00000100:2.0:1787829602.690971:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787829602.690981:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829602/real 1787829602] req@ffff8cb999f39a40 x1874672572622080/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829627 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829602.690992:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f39a40 x1874672572622080/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829627 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829602.691000:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829602.691003:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787829602.691006:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f39a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572622080:192.168.203.129@tcp:38:kworker.0 00010000:00010000:2.0:1787829603.522405:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829604.546468:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829605.570473:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829606.594448:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829607.618454:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787829607.810417:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829607.810425:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787829607.810429:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829607.810437:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 2559 00000100:00080000:3.0:1787829607.810460:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829607.810466:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:3.0:1787829607.810492:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb992316580 x1874672572622208/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787829607.810515:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829607.810518:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787829607.810520:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829607.810522:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 2559 00000100:00080000:3.0:1787829607.810524:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829607.810526:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:3.0:1787829607.810535:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787829607.810543:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb992317480 x1874672572622336/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787829607.810553:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829607.810555:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1787829607.810556:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829607.810561:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 2559 00000100:00080000:3.0:1787829607.810564:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1787829607.810565:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb992316580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572622208:192.168.203.129@tcp:250:kworker.0 00000100:00080000:3.0:1787829607.810567:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:3.0:1787829607.810572:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787829607.810579:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb992314b40 x1874672572622464/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1787829607.810640:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787829607.810673:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829607/real 1787829607] req@ffff8cb992316580 x1874672572622208/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829632 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829607.810699:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb992316580 x1874672572622208/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829632 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829607.810710:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829607.810715:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787829607.810719:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb992316580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572622208:192.168.203.129@tcp:250:kworker.0 00000100:00100000:2.0:1787829607.810745:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb992317480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572622336:192.168.203.129@tcp:38:kworker.0 00000800:00000100:2.0:1787829607.810772:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787829607.810786:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829607/real 1787829607] req@ffff8cb992317480 x1874672572622336/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829632 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829607.810800:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb992317480 x1874672572622336/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829632 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829607.810809:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829607.810813:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787829607.810817:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb992317480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572622336:192.168.203.129@tcp:38:kworker.0 00000100:00100000:2.0:1787829607.810824:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb992314b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572622464:192.168.203.129@tcp:38:kworker.0 00000800:00000100:2.0:1787829607.810844:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787829607.810854:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829607/real 1787829607] req@ffff8cb992314b40 x1874672572622464/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829632 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829607.810863:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb992314b40 x1874672572622464/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829632 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829607.810878:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829607.810880:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787829607.810883:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb992314b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572622464:192.168.203.129@tcp:38:kworker.0 00010000:00010000:2.0:1787829608.642438:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829609.666471:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829610.690415:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829611.714435:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829612.738402:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787829612.930395:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829612.930407:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787829612.930411:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829612.930421:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 2565 00000100:00080000:1.0:1787829612.930433:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829612.930440:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:1.0:1787829612.930477:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f38b40 x1874672572622592/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829612.930511:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829612.930516:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787829612.930518:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829612.930523:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 2565 00000100:00080000:1.0:1787829612.930527:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829612.930533:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:1.0:1787829612.930544:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787829612.930560:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f3b480 x1874672572622720/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787829612.930574:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f38b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572622592:192.168.203.129@tcp:250:kworker.0 00000100:00080000:1.0:1787829612.930603:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829612.930607:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1787829612.930609:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829612.930615:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 2565 00000100:00080000:1.0:1787829612.930620:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829612.930626:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:1.0:1787829612.930632:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787829612.930649:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f3b0c0 x1874672572622848/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1787829612.930664:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787829612.930723:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829612/real 1787829612] req@ffff8cb999f38b40 x1874672572622592/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829637 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829612.930746:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f38b40 x1874672572622592/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829637 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829612.930758:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829612.930762:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787829612.930766:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f38b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572622592:192.168.203.129@tcp:250:kworker.0 00000100:00100000:2.0:1787829612.930794:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f3b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572622720:192.168.203.129@tcp:38:kworker.0 00000800:00000100:2.0:1787829612.930814:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787829612.930824:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829612/real 1787829612] req@ffff8cb999f3b480 x1874672572622720/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829637 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829612.930832:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f3b480 x1874672572622720/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829637 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829612.930850:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829612.930852:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787829612.930855:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f3b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572622720:192.168.203.129@tcp:38:kworker.0 00000100:00100000:2.0:1787829612.930860:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f3b0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572622848:192.168.203.129@tcp:38:kworker.0 00000800:00000100:2.0:1787829612.930876:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787829612.930883:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829612/real 1787829612] req@ffff8cb999f3b0c0 x1874672572622848/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829637 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829612.930890:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f3b0c0 x1874672572622848/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829637 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829612.930898:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829612.930902:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787829612.930906:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f3b0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572622848:192.168.203.129@tcp:38:kworker.0 00010000:00010000:2.0:1787829613.762438:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829614.786394:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829615.810431:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829616.834422:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829617.858432:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787829618.050360:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829618.050386:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787829618.050391:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829618.050400:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 2570 00000100:00080000:3.0:1787829618.050412:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829618.050420:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:3.0:1787829618.050451:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb992314780 x1874672572622976/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787829618.050478:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829618.050482:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787829618.050484:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829618.050488:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 2570 00000100:00080000:3.0:1787829618.050492:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829618.050496:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:3.0:1787829618.050507:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787829618.050521:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb992315e00 x1874672572623104/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787829618.050530:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb992314780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572622976:192.168.203.129@tcp:250:kworker.0 00000100:00080000:3.0:1787829618.050538:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829618.050543:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1787829618.050545:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829618.050549:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 2570 00000100:00080000:3.0:1787829618.050554:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829618.050558:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:3.0:1787829618.050564:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787829618.050577:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb992316940 x1874672572623232/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1787829618.050601:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787829618.050656:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829618/real 1787829618] req@ffff8cb992314780 x1874672572622976/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829643 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829618.050679:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb992314780 x1874672572622976/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829643 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829618.050698:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829618.050703:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787829618.050707:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb992314780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572622976:192.168.203.129@tcp:250:kworker.0 00000100:00100000:2.0:1787829618.050731:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb992315e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572623104:192.168.203.129@tcp:38:kworker.0 00000800:00000100:2.0:1787829618.050758:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787829618.050771:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829618/real 1787829618] req@ffff8cb992315e00 x1874672572623104/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829643 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829618.050785:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb992315e00 x1874672572623104/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829643 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829618.050794:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829618.050797:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787829618.050800:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb992315e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572623104:192.168.203.129@tcp:38:kworker.0 00000100:00100000:2.0:1787829618.050807:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb992316940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572623232:192.168.203.129@tcp:38:kworker.0 00000800:00000100:2.0:1787829618.050830:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787829618.050842:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829618/real 1787829618] req@ffff8cb992316940 x1874672572623232/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829643 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829618.050854:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb992316940 x1874672572623232/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829643 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829618.050863:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829618.050866:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787829618.050870:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb992316940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572623232:192.168.203.129@tcp:38:kworker.0 00010000:00010000:2.0:1787829618.882392:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829619.906396:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829620.930402:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829621.954405:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829622.978419:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787829623.170382:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829623.170390:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787829623.170394:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829623.170402:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 2575 00000100:00080000:0.0:1787829623.170411:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829623.170417:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:0.0:1787829623.170448:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c6bc00 x1874672572623360/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829623.170472:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829623.170475:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787829623.170477:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829623.170479:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 2575 00000100:00080000:0.0:1787829623.170481:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829623.170484:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:0.0:1787829623.170491:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787829623.170500:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c69e00 x1874672572623488/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829623.170508:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829623.170510:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1787829623.170512:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829623.170517:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 2575 00000100:00100000:2.0:1787829623.170519:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c6bc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572623360:192.168.203.129@tcp:250:kworker.0 00000100:00080000:0.0:1787829623.170520:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829623.170527:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:0.0:1787829623.170544:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787829623.170555:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c6b480 x1874672572623616/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1787829623.170582:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787829623.170614:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829623/real 1787829623] req@ffff8cb999c6bc00 x1874672572623360/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829648 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829623.170628:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c6bc00 x1874672572623360/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829648 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829623.170636:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829623.170639:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787829623.170642:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c6bc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572623360:192.168.203.129@tcp:250:kworker.0 00000100:00100000:2.0:1787829623.170647:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c69e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572623488:192.168.203.129@tcp:38:kworker.0 00000800:00000100:2.0:1787829623.170661:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787829623.170669:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829623/real 1787829623] req@ffff8cb999c69e00 x1874672572623488/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829648 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829623.170677:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c69e00 x1874672572623488/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829648 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829623.170682:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829623.170684:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787829623.170686:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c69e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572623488:192.168.203.129@tcp:38:kworker.0 00000100:00100000:2.0:1787829623.170715:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c6b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572623616:192.168.203.129@tcp:38:kworker.0 00000800:00000100:2.0:1787829623.170742:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787829623.170754:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829623/real 1787829623] req@ffff8cb999c6b480 x1874672572623616/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829648 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829623.170767:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c6b480 x1874672572623616/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829648 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829623.170776:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829623.170780:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787829623.170783:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c6b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572623616:192.168.203.129@tcp:38:kworker.0 00010000:00010000:2.0:1787829624.002420:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829625.026414:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829626.050396:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829627.074399:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829628.098404:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787829628.290385:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829628.290396:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787829628.290400:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829628.290411:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 2580 00000100:00080000:3.0:1787829628.290424:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829628.290430:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:3.0:1787829628.290481:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb992314f00 x1874672572623744/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787829628.290511:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829628.290515:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787829628.290516:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829628.290519:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 2580 00000100:00080000:3.0:1787829628.290521:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829628.290524:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:3.0:1787829628.290533:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787829628.290542:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb992315a40 x1874672572623872/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787829628.290551:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829628.290554:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1787829628.290555:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829628.290560:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 2580 00000100:00100000:2.0:1787829628.290563:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb992314f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572623744:192.168.203.129@tcp:250:kworker.0 00000100:00080000:3.0:1787829628.290563:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829628.290570:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:3.0:1787829628.290593:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787829628.290604:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb9923170c0 x1874672572624000/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1787829628.290642:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787829628.290696:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829628/real 1787829628] req@ffff8cb992314f00 x1874672572623744/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829653 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829628.290713:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb992314f00 x1874672572623744/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829653 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829628.290729:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829628.290733:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787829628.290737:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb992314f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572623744:192.168.203.129@tcp:250:kworker.0 00000100:00100000:2.0:1787829628.290775:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb992315a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572623872:192.168.203.129@tcp:38:kworker.0 00000800:00000100:2.0:1787829628.290810:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787829628.290824:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829628/real 1787829628] req@ffff8cb992315a40 x1874672572623872/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829653 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829628.290838:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb992315a40 x1874672572623872/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829653 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829628.290847:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829628.290850:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787829628.290854:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb992315a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572623872:192.168.203.129@tcp:38:kworker.0 00000100:00100000:2.0:1787829628.290861:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb9923170c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572624000:192.168.203.129@tcp:38:kworker.0 00000800:00000100:2.0:1787829628.290885:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787829628.290896:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829628/real 1787829628] req@ffff8cb9923170c0 x1874672572624000/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829653 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829628.290908:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb9923170c0 x1874672572624000/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829653 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829628.290916:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829628.290919:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787829628.290923:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb9923170c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572624000:192.168.203.129@tcp:38:kworker.0 00010000:00010000:2.0:1787829629.122459:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829630.146414:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829631.170461:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829632.194475:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829633.218451:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787829633.410348:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829633.410359:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787829633.410364:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829633.410376:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 2585 00000100:00080000:1.0:1787829633.410388:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829633.410395:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:1.0:1787829633.410440:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f38f00 x1874672572624128/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829633.410468:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829633.410473:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787829633.410475:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829633.410479:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 2585 00000100:00080000:1.0:1787829633.410483:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829633.410487:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:1.0:1787829633.410526:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787829633.410530:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f38f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572624128:192.168.203.129@tcp:250:kworker.0 00000100:00100000:1.0:1787829633.410540:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f392c0 x1874672572624256/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829633.410551:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829633.410554:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1787829633.410557:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829633.410561:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 2585 00000100:00080000:1.0:1787829633.410565:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829633.410569:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:1.0:1787829633.410574:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787829633.410582:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f3a580 x1874672572624384/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1787829633.410618:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787829633.410667:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829633/real 1787829633] req@ffff8cb999f38f00 x1874672572624128/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829658 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829633.410682:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f38f00 x1874672572624128/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829658 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829633.410691:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829633.410694:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787829633.410697:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f38f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572624128:192.168.203.129@tcp:250:kworker.0 00000100:00100000:2.0:1787829633.410724:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f392c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572624256:192.168.203.129@tcp:38:kworker.0 00000800:00000100:2.0:1787829633.410742:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787829633.410752:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829633/real 1787829633] req@ffff8cb999f392c0 x1874672572624256/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829658 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829633.410760:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f392c0 x1874672572624256/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829658 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829633.410766:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829633.410769:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787829633.410771:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f392c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572624256:192.168.203.129@tcp:38:kworker.0 00000100:00100000:2.0:1787829633.410774:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f3a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572624384:192.168.203.129@tcp:38:kworker.0 00000800:00000100:2.0:1787829633.410785:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787829633.410792:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829633/real 1787829633] req@ffff8cb999f3a580 x1874672572624384/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829658 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829633.410807:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f3a580 x1874672572624384/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829658 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829633.410813:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829633.410816:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787829633.410818:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f3a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572624384:192.168.203.129@tcp:38:kworker.0 00010000:00010000:2.0:1787829634.242524:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829635.266454:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829636.290413:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829637.314465:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829638.338447:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787829638.530479:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829638.530488:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787829638.530494:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829638.530506:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 2590 00000100:00080000:0.0:1787829638.530519:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829638.530527:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:0.0:1787829638.530582:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c692c0 x1874672572624512/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829638.530634:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829638.530638:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787829638.530640:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829638.530644:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 2590 00000100:00080000:0.0:1787829638.530647:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829638.530651:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:0.0:1787829638.530663:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787829638.530678:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c69680 x1874672572624640/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829638.530692:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1787829638.530694:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c692c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572624512:192.168.203.129@tcp:250:kworker.0 00000100:00080000:0.0:1787829638.530697:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1787829638.530703:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829638.530712:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 2590 00000100:00080000:0.0:1787829638.530724:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829638.530732:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:0.0:1787829638.530739:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787829638.530755:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c6a1c0 x1874672572624768/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1787829638.530820:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787829638.530866:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829638/real 1787829638] req@ffff8cb999c692c0 x1874672572624512/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829663 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829638.530881:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c692c0 x1874672572624512/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829663 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829638.530890:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829638.530894:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787829638.530897:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c692c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572624512:192.168.203.129@tcp:250:kworker.0 00000100:00100000:1.0:1787829638.530927:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c69680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572624640:192.168.203.129@tcp:38:kworker.0 00000800:00000100:1.0:1787829638.530978:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787829638.530995:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829638/real 1787829638] req@ffff8cb999c69680 x1874672572624640/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829663 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829638.531022:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c69680 x1874672572624640/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829663 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829638.531029:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829638.531032:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787829638.531036:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c69680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572624640:192.168.203.129@tcp:38:kworker.0 00000100:00100000:1.0:1787829638.531056:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c6a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572624768:192.168.203.129@tcp:38:kworker.0 00000800:00000100:1.0:1787829638.531085:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787829638.531111:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829638/real 1787829638] req@ffff8cb999c6a1c0 x1874672572624768/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829663 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829638.531132:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c6a1c0 x1874672572624768/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829663 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829638.531145:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829638.531151:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787829638.531154:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c6a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572624768:192.168.203.129@tcp:38:kworker.0 00010000:00010000:2.0:1787829639.362501:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829640.386442:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829641.410422:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829642.434411:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829643.458449:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787829643.650402:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829643.650412:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787829643.650416:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829643.650424:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 2595 00000100:00080000:1.0:1787829643.650433:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829643.650455:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:1.0:1787829643.650484:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f383c0 x1874672572624896/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829643.650511:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829643.650514:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787829643.650515:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829643.650517:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 2595 00000100:00080000:1.0:1787829643.650520:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829643.650522:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:1.0:1787829643.650530:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787829643.650540:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f3a580 x1874672572625024/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829643.650547:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829643.650551:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1787829643.650552:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829643.650555:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 2595 00000100:00080000:1.0:1787829643.650557:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829643.650560:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:1.0:1787829643.650562:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787829643.650569:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f392c0 x1874672572625152/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787829643.650594:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f383c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572624896:192.168.203.129@tcp:250:kworker.0 00000100:00100000:2.0:1787829643.650731:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f3a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572625024:192.168.203.129@tcp:38:kworker.0 00000100:00100000:2.0:1787829643.650782:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f392c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572625152:192.168.203.129@tcp:38:kworker.0 00010000:00010000:2.0:1787829644.482490:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829645.506383:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829646.530771:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:1.0:1787829646.722591:0:2383:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.129:988 00000400:00000100:1.0:1787829646.722647:0:2383:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.129@tcp at host 192.168.203.129 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:1.0:1787829646.722690:0:2383:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.129@tcp 00000800:00000100:1.0:1787829646.722755:0:2383:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.129@tcp 00000800:00000100:1.0:1787829646.722768:0:2383:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.129@tcp 00000100:00100000:2.0:1787829646.722906:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829643/real 1787829646] req@ffff8cb999f392c0 x1874672572625152/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829668 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829646.722939:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f392c0 x1874672572625152/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829668 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829646.722965:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829646.722972:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787829646.722978:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f392c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572625152:192.168.203.129@tcp:38:kworker.0 00000100:00100000:2.0:1787829646.722988:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829643/real 1787829646] req@ffff8cb999f383c0 x1874672572624896/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829668 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829646.723004:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f383c0 x1874672572624896/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829668 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829646.723014:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829646.723018:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787829646.723021:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f383c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572624896:192.168.203.129@tcp:250:kworker.0 00000100:00100000:2.0:1787829646.723028:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829643/real 1787829646] req@ffff8cb999f3a580 x1874672572625024/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829668 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829646.723041:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f3a580 x1874672572625024/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829668 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829646.723049:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829646.723053:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787829646.723057:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f3a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572625024:192.168.203.129@tcp:38:kworker.0 00010000:00010000:2.0:1787829647.554620:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829648.578495:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787829648.770415:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829648.770423:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787829648.770427:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829648.770434:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 2600 00000100:00080000:1.0:1787829648.770447:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829648.770454:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:1.0:1787829648.770482:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f38f00 x1874672572625280/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829648.770503:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829648.770506:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787829648.770507:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829648.770510:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 2600 00000100:00080000:1.0:1787829648.770512:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829648.770514:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:1.0:1787829648.770522:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787829648.770530:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f3b0c0 x1874672572625408/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829648.770537:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829648.770539:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1787829648.770540:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829648.770543:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 2600 00000100:00080000:1.0:1787829648.770557:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829648.770560:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:1.0:1787829648.770564:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787829648.770573:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f3b480 x1874672572625536/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787829648.770616:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f38f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572625280:192.168.203.129@tcp:250:kworker.0 00000800:00000100:2.0:1787829648.770716:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787829648.770806:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829648/real 1787829648] req@ffff8cb999f38f00 x1874672572625280/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829673 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829648.770832:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f38f00 x1874672572625280/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829673 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829648.770848:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829648.770854:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787829648.770859:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f38f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572625280:192.168.203.129@tcp:250:kworker.0 00000100:00100000:2.0:1787829648.770869:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f3b0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572625408:192.168.203.129@tcp:38:kworker.0 00000800:00000100:2.0:1787829648.770899:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787829648.770914:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829648/real 1787829648] req@ffff8cb999f3b0c0 x1874672572625408/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829673 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829648.770928:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f3b0c0 x1874672572625408/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829673 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829648.770938:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829648.770942:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787829648.770946:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f3b0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572625408:192.168.203.129@tcp:38:kworker.0 00000100:00100000:2.0:1787829648.770953:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f3b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572625536:192.168.203.129@tcp:38:kworker.0 00000800:00000100:2.0:1787829648.770980:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787829648.770992:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829648/real 1787829648] req@ffff8cb999f3b480 x1874672572625536/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829673 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829648.771004:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f3b480 x1874672572625536/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829673 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829648.771026:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829648.771030:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787829648.771034:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f3b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572625536:192.168.203.129@tcp:38:kworker.0 00010000:00010000:2.0:1787829649.602467:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829650.627137:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829651.650571:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829652.674835:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829653.698695:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787829653.890381:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829653.890390:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787829653.890393:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829653.890402:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 2606 00000100:00080000:2.0:1787829653.890412:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829653.890418:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:2.0:1787829653.890449:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8e580 x1874672572625664/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829653.890483:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829653.890488:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787829653.890490:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829653.890494:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 2606 00000100:00080000:2.0:1787829653.890498:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829653.890502:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:2.0:1787829653.890521:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787829653.890530:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8e580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572625664:192.168.203.129@tcp:250:kworker.0 00000100:00100000:2.0:1787829653.890535:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8d680 x1874672572625792/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829653.890559:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829653.890562:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1787829653.890563:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829653.890567:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 2606 00000100:00080000:2.0:1787829653.890571:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829653.890575:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:2.0:1787829653.890581:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000800:00000100:1.0:1787829653.890587:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787829653.890596:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8f480 x1874672572625920/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787829653.890627:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829653/real 1787829653] req@ffff8cb998a8e580 x1874672572625664/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829678 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829653.890642:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8e580 x1874672572625664/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829678 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829653.890649:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829653.890652:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787829653.890655:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8e580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572625664:192.168.203.129@tcp:250:kworker.0 00000100:00100000:1.0:1787829653.890673:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8d680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572625792:192.168.203.129@tcp:38:kworker.0 00000800:00000100:1.0:1787829653.890688:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787829653.890696:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829653/real 1787829653] req@ffff8cb998a8d680 x1874672572625792/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829678 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829653.890704:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8d680 x1874672572625792/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829678 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829653.890710:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829653.890712:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787829653.890720:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8d680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572625792:192.168.203.129@tcp:38:kworker.0 00000100:00100000:1.0:1787829653.890725:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572625920:192.168.203.129@tcp:38:kworker.0 00000800:00000100:1.0:1787829653.890744:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787829653.890756:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829653/real 1787829653] req@ffff8cb998a8f480 x1874672572625920/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829678 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829653.890769:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8f480 x1874672572625920/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829678 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829653.890778:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829653.890782:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787829653.890785:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572625920:192.168.203.129@tcp:38:kworker.0 00010000:00010000:2.0:1787829654.722904:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829655.747333:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829656.770480:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829657.794456:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829658.818445:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787829659.010384:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829659.010395:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787829659.010399:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829659.010409:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 2611 00000100:00080000:2.0:1787829659.010421:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829659.010427:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:2.0:1787829659.010474:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8c000 x1874672572626048/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829659.010511:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829659.010514:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787829659.010536:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829659.010539:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 2611 00000100:00080000:2.0:1787829659.010544:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829659.010549:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:2.0:1787829659.010562:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787829659.010570:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572626048:192.168.203.129@tcp:250:kworker.0 00000100:00100000:2.0:1787829659.010579:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8cb40 x1874672572626176/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829659.010595:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829659.010597:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1787829659.010599:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829659.010602:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 2611 00000100:00080000:2.0:1787829659.010606:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829659.010609:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:2.0:1787829659.010614:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787829659.010623:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8ed00 x1874672572626304/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1787829659.010643:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787829659.010688:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829659/real 1787829659] req@ffff8cb998a8c000 x1874672572626048/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829684 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829659.010702:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8c000 x1874672572626048/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829684 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829659.010711:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829659.010714:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787829659.010718:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572626048:192.168.203.129@tcp:250:kworker.0 00000100:00100000:1.0:1787829659.010740:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8cb40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572626176:192.168.203.129@tcp:38:kworker.0 00000800:00000100:1.0:1787829659.010754:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787829659.010771:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829659/real 1787829659] req@ffff8cb998a8cb40 x1874672572626176/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829684 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829659.010780:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8cb40 x1874672572626176/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829684 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829659.010786:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829659.010788:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787829659.010790:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8cb40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572626176:192.168.203.129@tcp:38:kworker.0 00000100:00100000:1.0:1787829659.010794:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8ed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572626304:192.168.203.129@tcp:38:kworker.0 00000800:00000100:1.0:1787829659.010807:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787829659.010813:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829659/real 1787829659] req@ffff8cb998a8ed00 x1874672572626304/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829684 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829659.010821:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8ed00 x1874672572626304/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829684 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829659.010826:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829659.010827:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787829659.010829:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8ed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572626304:192.168.203.129@tcp:38:kworker.0 00010000:00010000:2.0:1787829659.842411:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829660.866458:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829661.890416:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829662.914384:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787829663.938469:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787829664.130343:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829664.130351:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787829664.130354:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829664.130362:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 2616 00000100:00080000:2.0:1787829664.130371:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829664.130392:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:2.0:1787829664.130421:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8e940 x1874672572626432/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829664.130444:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829664.130452:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787829664.130453:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829664.130458:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 2616 00000100:00080000:2.0:1787829664.130462:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829664.130464:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:2.0:1787829664.130473:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787829664.130482:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8f840 x1874672572626560/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829664.130490:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829664.130492:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1787829664.130494:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1787829664.130498:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8e940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572626432:192.168.203.129@tcp:250:kworker.0 00000100:00080000:2.0:1787829664.130499:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 2616 00000100:00080000:2.0:1787829664.130505:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829664.130508:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:2.0:1787829664.130512:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787829664.130525:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8ed00 x1874672572626688/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1787829664.130564:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787829664.130596:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829664/real 1787829664] req@ffff8cb998a8e940 x1874672572626432/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829689 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829664.130609:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8e940 x1874672572626432/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829689 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829664.130630:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829664.130634:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787829664.130637:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8e940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572626432:192.168.203.129@tcp:250:kworker.0 00000100:00100000:1.0:1787829664.130661:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8f840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572626560:192.168.203.129@tcp:38:kworker.0 00000800:00000100:1.0:1787829664.130679:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787829664.130689:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829664/real 1787829664] req@ffff8cb998a8f840 x1874672572626560/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829689 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829664.130697:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8f840 x1874672572626560/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829689 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829664.130703:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829664.130705:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787829664.130709:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8f840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572626560:192.168.203.129@tcp:38:kworker.0 00000100:00100000:1.0:1787829664.130715:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8ed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572626688:192.168.203.129@tcp:38:kworker.0 00000800:00000100:1.0:1787829664.130735:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787829664.130747:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829664/real 1787829664] req@ffff8cb998a8ed00 x1874672572626688/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829689 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829664.130761:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8ed00 x1874672572626688/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829689 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829664.130768:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829664.130770:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787829664.130772:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8ed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572626688:192.168.203.129@tcp:38:kworker.0 00010000:00010000:2.0:1787829664.962402:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829665.986474:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787829667.010403:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829668.034428:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829669.058416:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787829669.250375:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829669.250384:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787829669.250387:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829669.250396:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 2621 00000100:00080000:1.0:1787829669.250404:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829669.250411:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:1.0:1787829669.250447:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f38b40 x1874672572626816/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829669.250477:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829669.250481:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787829669.250483:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829669.250487:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 2621 00000100:00080000:1.0:1787829669.250490:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829669.250494:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:1.0:1787829669.250502:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787829669.250508:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f38b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572626816:192.168.203.129@tcp:250:kworker.0 00000100:00100000:1.0:1787829669.250516:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f39a40 x1874672572626944/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829669.250526:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829669.250528:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1787829669.250529:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829669.250532:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 2621 00000100:00080000:1.0:1787829669.250534:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829669.250537:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:1.0:1787829669.250554:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787829669.250566:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f39e00 x1874672572627072/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1787829669.250580:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787829669.250618:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829669/real 1787829669] req@ffff8cb999f38b40 x1874672572626816/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829694 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829669.250632:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f38b40 x1874672572626816/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829694 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829669.250640:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829669.250643:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787829669.250646:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f38b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572626816:192.168.203.129@tcp:250:kworker.0 00000100:00100000:2.0:1787829669.250669:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f39a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572626944:192.168.203.129@tcp:38:kworker.0 00000800:00000100:2.0:1787829669.250694:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787829669.250707:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829669/real 1787829669] req@ffff8cb999f39a40 x1874672572626944/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829694 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829669.250716:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f39a40 x1874672572626944/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829694 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829669.250722:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829669.250724:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787829669.250726:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f39a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572626944:192.168.203.129@tcp:38:kworker.0 00000100:00100000:2.0:1787829669.250731:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f39e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572627072:192.168.203.129@tcp:38:kworker.0 00000800:00000100:2.0:1787829669.250744:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787829669.250759:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829669/real 1787829669] req@ffff8cb999f39e00 x1874672572627072/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829694 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829669.250768:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f39e00 x1874672572627072/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829694 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829669.250774:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829669.250776:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787829669.250778:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f39e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572627072:192.168.203.129@tcp:38:kworker.0 00010000:00010000:0.0:1787829670.082446:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787829671.106475:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829672.130433:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787829673.154448:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787829674.178420:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787829674.370367:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829674.370375:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787829674.370380:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829674.370389:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 2626 00000100:00080000:1.0:1787829674.370401:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829674.370409:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:1.0:1787829674.370446:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f3b840 x1874672572627200/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829674.370469:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829674.370472:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787829674.370474:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829674.370476:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 2626 00000100:00080000:1.0:1787829674.370479:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829674.370482:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:1.0:1787829674.370489:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787829674.370503:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f39e00 x1874672572627328/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829674.370514:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829674.370518:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1787829674.370520:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829674.370526:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 2626 00000100:00080000:1.0:1787829674.370544:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829674.370557:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:2.0:1787829674.370559:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f3b840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572627200:192.168.203.129@tcp:250:kworker.0 00000008:00100000:1.0:1787829674.370563:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787829674.370577:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f39a40 x1874672572627456/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1787829674.370635:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787829674.370680:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829674/real 1787829674] req@ffff8cb999f3b840 x1874672572627200/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829699 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829674.370699:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f3b840 x1874672572627200/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829699 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829674.370710:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829674.370714:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787829674.370719:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f3b840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572627200:192.168.203.129@tcp:250:kworker.0 00000100:00100000:2.0:1787829674.370727:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f39e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572627328:192.168.203.129@tcp:38:kworker.0 00000800:00000100:2.0:1787829674.370753:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787829674.370770:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829674/real 1787829674] req@ffff8cb999f39e00 x1874672572627328/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829699 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829674.370792:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f39e00 x1874672572627328/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829699 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829674.370808:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829674.370813:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787829674.370818:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f39e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572627328:192.168.203.129@tcp:38:kworker.0 00000100:00100000:2.0:1787829674.370850:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f39a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572627456:192.168.203.129@tcp:38:kworker.0 00000800:00000100:2.0:1787829674.370878:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787829674.370896:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829674/real 1787829674] req@ffff8cb999f39a40 x1874672572627456/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829699 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829674.370908:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f39a40 x1874672572627456/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829699 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829674.370935:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829674.370938:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787829674.370942:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f39a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572627456:192.168.203.129@tcp:38:kworker.0 00010000:00010000:2.0:1787829675.202422:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787829676.226428:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787829677.250458:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829678.274450:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787829679.298436:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787829679.490418:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829679.490431:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787829679.490437:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829679.490449:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 2631 00000100:00080000:2.0:1787829679.490464:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829679.490473:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:2.0:1787829679.490521:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8cb40 x1874672572627584/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829679.490583:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829679.490587:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787829679.490589:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829679.490602:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 2631 00000100:00080000:2.0:1787829679.490608:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829679.490614:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:2.0:1787829679.490626:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787829679.490637:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8cb40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572627584:192.168.203.129@tcp:250:kworker.0 00000100:00100000:2.0:1787829679.490643:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8c000 x1874672572627712/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829679.490656:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829679.490660:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1787829679.490662:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829679.490666:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 2631 00000100:00080000:2.0:1787829679.490669:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829679.490674:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:2.0:1787829679.490679:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787829679.490691:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8f480 x1874672572627840/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1787829679.490719:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787829679.490770:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829679/real 1787829679] req@ffff8cb998a8cb40 x1874672572627584/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829704 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829679.490786:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8cb40 x1874672572627584/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829704 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829679.490795:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829679.490798:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787829679.490801:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8cb40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572627584:192.168.203.129@tcp:250:kworker.0 00000100:00100000:1.0:1787829679.490822:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572627712:192.168.203.129@tcp:38:kworker.0 00000800:00000100:1.0:1787829679.490841:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787829679.490850:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829679/real 1787829679] req@ffff8cb998a8c000 x1874672572627712/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829704 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829679.490859:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8c000 x1874672572627712/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829704 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829679.490864:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829679.490867:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787829679.490875:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572627712:192.168.203.129@tcp:38:kworker.0 00000100:00100000:1.0:1787829679.490880:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572627840:192.168.203.129@tcp:38:kworker.0 00000800:00000100:1.0:1787829679.490912:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787829679.490921:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829679/real 1787829679] req@ffff8cb998a8f480 x1874672572627840/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829704 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829679.490929:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8f480 x1874672572627840/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829704 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829679.490935:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829679.490937:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787829679.490939:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572627840:192.168.203.129@tcp:38:kworker.0 00010000:00010000:0.0:1787829680.322495:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829681.346445:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787829682.370441:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787829683.394449:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829684.418439:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787829684.610441:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829684.610457:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787829684.610461:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829684.610484:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 2636 00000100:00080000:1.0:1787829684.610500:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829684.610509:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:1.0:1787829684.610574:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f38b40 x1874672572627968/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829684.610613:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829684.610618:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787829684.610621:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829684.610626:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 2636 00000100:00080000:1.0:1787829684.610631:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829684.610656:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:2.0:1787829684.610662:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f38b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572627968:192.168.203.129@tcp:250:kworker.0 00000008:00100000:1.0:1787829684.610672:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787829684.610708:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f3b480 x1874672572628096/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829684.610720:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829684.610724:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1787829684.610726:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829684.610731:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 2636 00000100:00080000:1.0:1787829684.610736:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829684.610741:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:1.0:1787829684.610747:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787829684.610777:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f3b0c0 x1874672572628224/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1787829684.610780:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787829684.610869:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829684/real 1787829684] req@ffff8cb999f38b40 x1874672572627968/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829709 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829684.610906:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f38b40 x1874672572627968/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829709 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829684.610916:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829684.610920:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787829684.610923:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f38b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572627968:192.168.203.129@tcp:250:kworker.0 00000100:00100000:2.0:1787829684.610952:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f3b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572628096:192.168.203.129@tcp:38:kworker.0 00000800:00000100:2.0:1787829684.610972:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787829684.610981:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829684/real 1787829684] req@ffff8cb999f3b480 x1874672572628096/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829709 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829684.610989:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f3b480 x1874672572628096/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829709 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829684.610995:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829684.610997:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787829684.611000:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f3b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572628096:192.168.203.129@tcp:38:kworker.0 00000100:00100000:2.0:1787829684.611003:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f3b0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572628224:192.168.203.129@tcp:38:kworker.0 00000800:00000100:2.0:1787829684.611015:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787829684.611021:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829684/real 1787829684] req@ffff8cb999f3b0c0 x1874672572628224/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829709 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829684.611028:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f3b0c0 x1874672572628224/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829709 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829684.611043:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829684.611046:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787829684.611048:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f3b0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572628224:192.168.203.129@tcp:38:kworker.0 00010000:00010000:0.0:1787829685.442455:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787829686.466412:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829687.490422:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787829688.514488:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787829689.538475:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787829689.730376:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829689.730388:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787829689.730393:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829689.730404:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 2641 00000100:00080000:0.0:1787829689.730417:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829689.730426:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:0.0:1787829689.730492:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c68000 x1874672572628352/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829689.730560:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829689.730563:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787829689.730564:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829689.730568:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 2641 00000100:00080000:0.0:1787829689.730571:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829689.730575:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:0.0:1787829689.730605:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787829689.730612:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c68000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572628352:192.168.203.129@tcp:250:kworker.0 00000100:00100000:0.0:1787829689.730619:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c6a580 x1874672572628480/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829689.730628:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829689.730631:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1787829689.730633:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829689.730635:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 2641 00000100:00080000:0.0:1787829689.730638:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829689.730641:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:0.0:1787829689.730655:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787829689.730665:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c6a1c0 x1874672572628608/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1787829689.730697:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787829689.730766:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829689/real 1787829689] req@ffff8cb999c68000 x1874672572628352/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829714 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829689.730793:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c68000 x1874672572628352/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829714 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829689.730809:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829689.730829:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787829689.730834:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c68000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572628352:192.168.203.129@tcp:250:kworker.0 00000100:00100000:2.0:1787829689.730890:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c6a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572628480:192.168.203.129@tcp:38:kworker.0 00000800:00000100:2.0:1787829689.730910:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787829689.730920:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829689/real 1787829689] req@ffff8cb999c6a580 x1874672572628480/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829714 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829689.730928:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c6a580 x1874672572628480/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829714 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829689.730934:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829689.730937:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787829689.730945:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c6a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572628480:192.168.203.129@tcp:38:kworker.0 00000100:00100000:2.0:1787829689.730950:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c6a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572628608:192.168.203.129@tcp:38:kworker.0 00000800:00000100:2.0:1787829689.730965:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787829689.730972:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829689/real 1787829689] req@ffff8cb999c6a1c0 x1874672572628608/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829714 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829689.730981:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c6a1c0 x1874672572628608/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829714 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829689.730986:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829689.730988:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787829689.730990:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c6a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572628608:192.168.203.129@tcp:38:kworker.0 00010000:00010000:2.0:1787829690.562460:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787829691.586428:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787829692.610421:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829693.634463:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787829694.658396:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787829694.850447:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829694.850458:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787829694.850462:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829694.850473:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 2647 00000100:00080000:2.0:1787829694.850485:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829694.850496:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:2.0:1787829694.850540:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8d680 x1874672572628736/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829694.850571:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829694.850577:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787829694.850579:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829694.850599:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 2647 00000100:00080000:2.0:1787829694.850603:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829694.850607:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:2.0:1787829694.850617:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787829694.850627:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8d680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572628736:192.168.203.129@tcp:250:kworker.0 00000100:00100000:2.0:1787829694.850629:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8e580 x1874672572628864/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829694.850643:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829694.850646:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1787829694.850647:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829694.850650:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 2647 00000100:00080000:2.0:1787829694.850652:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829694.850654:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:2.0:1787829694.850657:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787829694.850666:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8cf00 x1874672572628992/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1787829694.850710:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787829694.850756:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829694/real 1787829694] req@ffff8cb998a8d680 x1874672572628736/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829719 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829694.850787:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8d680 x1874672572628736/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829719 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829694.850810:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829694.850815:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787829694.850820:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8d680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572628736:192.168.203.129@tcp:250:kworker.0 00000100:00100000:1.0:1787829694.850853:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8e580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572628864:192.168.203.129@tcp:38:kworker.0 00000800:00000100:1.0:1787829694.850874:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787829694.850884:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829694/real 1787829694] req@ffff8cb998a8e580 x1874672572628864/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829719 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829694.850893:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8e580 x1874672572628864/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829719 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829694.850899:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829694.850902:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787829694.850904:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8e580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572628864:192.168.203.129@tcp:38:kworker.0 00000100:00100000:1.0:1787829694.850908:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8cf00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572628992:192.168.203.129@tcp:38:kworker.0 00000800:00000100:1.0:1787829694.850920:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787829694.850926:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829694/real 1787829694] req@ffff8cb998a8cf00 x1874672572628992/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829719 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829694.850934:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8cf00 x1874672572628992/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829719 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829694.850939:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829694.850941:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787829694.850943:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8cf00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572628992:192.168.203.129@tcp:38:kworker.0 00010000:00010000:0.0:1787829695.682378:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829696.706433:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787829697.730445:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787829698.754430:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829699.778388:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787829699.970386:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829699.970395:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787829699.970398:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829699.970407:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 2652 00000100:00080000:0.0:1787829699.970416:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829699.970422:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:0.0:1787829699.970465:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c69680 x1874672572629120/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829699.970489:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829699.970491:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787829699.970493:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829699.970495:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 2652 00000100:00080000:0.0:1787829699.970498:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829699.970500:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:0.0:1787829699.970507:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787829699.970515:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c692c0 x1874672572629248/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829699.970522:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829699.970525:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1787829699.970526:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829699.970529:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 2652 00000100:00080000:0.0:1787829699.970531:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829699.970533:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:0.0:1787829699.970549:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787829699.970556:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c6b480 x1874672572629376/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787829699.970581:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c69680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572629120:192.168.203.129@tcp:250:kworker.0 00000800:00000100:1.0:1787829699.970687:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787829699.970760:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829699/real 1787829699] req@ffff8cb999c69680 x1874672572629120/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829724 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829699.970797:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c69680 x1874672572629120/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829724 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829699.970811:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829699.970817:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787829699.970821:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c69680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572629120:192.168.203.129@tcp:250:kworker.0 00000100:00100000:1.0:1787829699.970832:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c692c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572629248:192.168.203.129@tcp:38:kworker.0 00000800:00000100:1.0:1787829699.970860:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787829699.970876:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829699/real 1787829699] req@ffff8cb999c692c0 x1874672572629248/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829724 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829699.970890:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c692c0 x1874672572629248/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829724 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829699.970899:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829699.970903:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787829699.970907:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c692c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572629248:192.168.203.129@tcp:38:kworker.0 00000100:00100000:1.0:1787829699.970938:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c6b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572629376:192.168.203.129@tcp:38:kworker.0 00000800:00000100:1.0:1787829699.970960:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787829699.970973:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829699/real 1787829699] req@ffff8cb999c6b480 x1874672572629376/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829724 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829699.970987:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c6b480 x1874672572629376/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829724 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829699.971009:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829699.971014:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787829699.971018:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c6b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572629376:192.168.203.129@tcp:38:kworker.0 00010000:00010000:2.0:1787829700.802519:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787829701.826358:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829702.850411:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829703.874441:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787829704.898464:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787829705.090380:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829705.090394:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787829705.090399:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829705.090412:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 2657 00000100:00080000:1.0:1787829705.090427:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829705.090434:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:1.0:1787829705.090476:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f38f00 x1874672572629504/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829705.090507:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829705.090510:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787829705.090512:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829705.090514:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 2657 00000100:00080000:1.0:1787829705.090517:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829705.090520:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:1.0:1787829705.090530:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787829705.090540:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f3a580 x1874672572629632/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829705.090549:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829705.090552:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00100000:2.0:1787829705.090561:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f38f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572629504:192.168.203.129@tcp:250:kworker.0 00000100:00080000:1.0:1787829705.090573:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829705.090578:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 2657 00000100:00080000:1.0:1787829705.090581:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829705.090585:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:1.0:1787829705.090590:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787829705.090599:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f383c0 x1874672572629760/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1787829705.090642:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787829705.090689:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829705/real 1787829705] req@ffff8cb999f38f00 x1874672572629504/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829730 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829705.090745:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f38f00 x1874672572629504/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829730 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829705.090756:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829705.090759:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787829705.090762:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f38f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572629504:192.168.203.129@tcp:250:kworker.0 00000100:00100000:2.0:1787829705.090786:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f3a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572629632:192.168.203.129@tcp:38:kworker.0 00000800:00000100:2.0:1787829705.090811:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787829705.090826:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829705/real 1787829705] req@ffff8cb999f3a580 x1874672572629632/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829730 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829705.090839:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f3a580 x1874672572629632/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829730 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829705.090845:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829705.090847:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787829705.090850:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f3a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572629632:192.168.203.129@tcp:38:kworker.0 00000100:00100000:2.0:1787829705.090855:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f383c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572629760:192.168.203.129@tcp:38:kworker.0 00000800:00000100:2.0:1787829705.090871:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787829705.090878:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829705/real 1787829705] req@ffff8cb999f383c0 x1874672572629760/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829730 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829705.090886:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f383c0 x1874672572629760/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829730 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829705.090901:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829705.090903:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787829705.090905:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f383c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572629760:192.168.203.129@tcp:38:kworker.0 00010000:00010000:2.0:1787829705.922407:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829706.946414:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787829707.970394:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829708.994481:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829710.018468:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787829710.210393:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829710.210406:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787829710.210410:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829710.210423:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 2662 00000100:00080000:1.0:1787829710.210436:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829710.210444:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:1.0:1787829710.210495:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f392c0 x1874672572629888/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829710.210539:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829710.210542:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787829710.210544:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829710.210549:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 2662 00000100:00080000:1.0:1787829710.210553:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829710.210557:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:1.0:1787829710.210568:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787829710.210583:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f383c0 x1874672572630016/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787829710.210593:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f392c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572629888:192.168.203.129@tcp:250:kworker.0 00000100:00080000:1.0:1787829710.210609:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829710.210612:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1787829710.210614:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829710.210618:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 2662 00000100:00080000:1.0:1787829710.210622:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829710.210625:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:1.0:1787829710.210630:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787829710.210640:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f3a580 x1874672572630144/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787829710.210723:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f383c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572630016:192.168.203.129@tcp:38:kworker.0 00000100:00100000:0.0:1787829710.210755:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f3a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572630144:192.168.203.129@tcp:38:kworker.0 00010000:00010000:0.0:1787829711.042651:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829712.066450:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829713.090456:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:1.0:1787829713.282515:0:2384:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.129:988 00000400:00000100:1.0:1787829713.282553:0:2384:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.129@tcp at host 192.168.203.129 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:1.0:1787829713.282570:0:2384:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.129@tcp 00000800:00000100:1.0:1787829713.282611:0:2384:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.129@tcp 00000800:00000100:1.0:1787829713.282621:0:2384:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.129@tcp 00000100:00100000:0.0:1787829713.282683:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829710/real 1787829713] req@ffff8cb999f383c0 x1874672572630016/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829735 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829713.282711:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f383c0 x1874672572630016/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829735 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829713.282722:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829713.282727:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787829713.282732:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f383c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572630016:192.168.203.129@tcp:38:kworker.0 00000100:00100000:0.0:1787829713.282740:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829710/real 1787829713] req@ffff8cb999f3a580 x1874672572630144/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829735 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829713.282752:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f3a580 x1874672572630144/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829735 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829713.282760:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829713.282780:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787829713.282785:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f3a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572630144:192.168.203.129@tcp:38:kworker.0 00000100:00100000:0.0:1787829713.282793:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829710/real 1787829713] req@ffff8cb999f392c0 x1874672572629888/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829735 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829713.282809:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f392c0 x1874672572629888/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829735 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829713.282818:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829713.282821:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787829713.282825:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f392c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572629888:192.168.203.129@tcp:250:kworker.0 00010000:00010000:2.0:1787829714.114418:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829715.138451:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787829715.330408:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829715.330418:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787829715.330422:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829715.330430:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 2667 00000100:00080000:1.0:1787829715.330441:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829715.330448:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:1.0:1787829715.330480:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f38f00 x1874672572630272/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829715.330506:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829715.330523:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787829715.330526:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829715.330531:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 2667 00000100:00080000:1.0:1787829715.330535:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829715.330540:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:0.0:1787829715.330547:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f38f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572630272:192.168.203.129@tcp:250:kworker.0 00000008:00100000:1.0:1787829715.330550:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787829715.330567:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f3b0c0 x1874672572630400/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829715.330579:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829715.330583:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1787829715.330585:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829715.330589:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 2667 00000100:00080000:1.0:1787829715.330609:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829715.330613:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:1.0:1787829715.330617:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000800:00000100:0.0:1787829715.330622:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787829715.330645:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f3b480 x1874672572630528/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787829715.330681:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829715/real 1787829715] req@ffff8cb999f38f00 x1874672572630272/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829740 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829715.330701:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f38f00 x1874672572630272/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829740 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829715.330712:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829715.330716:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787829715.330720:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f38f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572630272:192.168.203.129@tcp:250:kworker.0 00000100:00100000:0.0:1787829715.330747:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f3b0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572630400:192.168.203.129@tcp:38:kworker.0 00000800:00000100:0.0:1787829715.330773:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787829715.330788:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829715/real 1787829715] req@ffff8cb999f3b0c0 x1874672572630400/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829740 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829715.330819:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f3b0c0 x1874672572630400/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829740 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829715.330828:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829715.330831:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787829715.330835:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f3b0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572630400:192.168.203.129@tcp:38:kworker.0 00000100:00100000:0.0:1787829715.330842:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f3b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572630528:192.168.203.129@tcp:38:kworker.0 00000800:00000100:0.0:1787829715.330866:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787829715.330879:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829715/real 1787829715] req@ffff8cb999f3b480 x1874672572630528/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829740 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829715.330893:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f3b480 x1874672572630528/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829740 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829715.330901:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829715.330905:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787829715.330908:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f3b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572630528:192.168.203.129@tcp:38:kworker.0 00010000:00010000:2.0:1787829716.162432:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829717.186456:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829718.210542:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829719.234995:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829720.258431:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787829720.450411:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829720.450426:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787829720.450431:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829720.450444:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 2672 00000100:00080000:2.0:1787829720.450457:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829720.450463:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:2.0:1787829720.450508:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8c780 x1874672572630656/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829720.450537:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829720.450541:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787829720.450558:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829720.450562:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 2672 00000100:00080000:2.0:1787829720.450565:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829720.450569:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:2.0:1787829720.450579:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787829720.450590:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8f0c0 x1874672572630784/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829720.450613:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829720.450616:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1787829720.450618:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829720.450626:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 2672 00000100:00100000:0.0:1787829720.450627:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8c780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572630656:192.168.203.129@tcp:250:kworker.0 00000100:00080000:2.0:1787829720.450630:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829720.450638:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:2.0:1787829720.450646:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787829720.450657:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8fc00 x1874672572630912/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1787829720.450718:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787829720.450768:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829720/real 1787829720] req@ffff8cb998a8c780 x1874672572630656/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829745 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829720.450789:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8c780 x1874672572630656/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829745 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829720.450813:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829720.450817:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787829720.450820:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8c780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572630656:192.168.203.129@tcp:250:kworker.0 00000100:00100000:0.0:1787829720.450843:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8f0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572630784:192.168.203.129@tcp:38:kworker.0 00000800:00000100:0.0:1787829720.450864:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787829720.450874:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829720/real 1787829720] req@ffff8cb998a8f0c0 x1874672572630784/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829745 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829720.450888:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8f0c0 x1874672572630784/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829745 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829720.450894:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829720.450896:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787829720.450899:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8f0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572630784:192.168.203.129@tcp:38:kworker.0 00000100:00100000:0.0:1787829720.450903:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8fc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572630912:192.168.203.129@tcp:38:kworker.0 00000800:00000100:0.0:1787829720.450918:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787829720.450924:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829720/real 1787829720] req@ffff8cb998a8fc00 x1874672572630912/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829745 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829720.450933:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8fc00 x1874672572630912/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829745 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829720.450941:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829720.450945:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787829720.450948:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8fc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572630912:192.168.203.129@tcp:38:kworker.0 00010000:00010000:2.0:1787829721.282471:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829722.306769:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829723.330455:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829724.354504:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829725.378425:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787829725.570386:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829725.570396:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787829725.570401:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829725.570412:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 2677 00000100:00080000:0.0:1787829725.570424:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829725.570431:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:0.0:1787829725.570486:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c69e00 x1874672572631040/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829725.570513:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829725.570516:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787829725.570517:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829725.570520:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 2677 00000100:00080000:0.0:1787829725.570523:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829725.570526:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:0.0:1787829725.570536:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787829725.570553:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c6bc00 x1874672572631168/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787829725.570574:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c69e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572631040:192.168.203.129@tcp:250:kworker.0 00000100:00080000:0.0:1787829725.570578:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829725.570584:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1787829725.570585:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829725.570589:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 2677 00000100:00080000:0.0:1787829725.570592:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829725.570595:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:0.0:1787829725.570619:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787829725.570631:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c68f00 x1874672572631296/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1787829725.570668:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787829725.570719:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829725/real 1787829725] req@ffff8cb999c69e00 x1874672572631040/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829750 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829725.570749:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c69e00 x1874672572631040/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829750 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829725.570758:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829725.570761:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787829725.570764:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c69e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572631040:192.168.203.129@tcp:250:kworker.0 00000100:00100000:2.0:1787829725.570789:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c6bc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572631168:192.168.203.129@tcp:38:kworker.0 00000800:00000100:2.0:1787829725.570808:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787829725.570817:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829725/real 1787829725] req@ffff8cb999c6bc00 x1874672572631168/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829750 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829725.570826:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c6bc00 x1874672572631168/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829750 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829725.570832:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829725.570834:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787829725.570836:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c6bc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572631168:192.168.203.129@tcp:38:kworker.0 00000100:00100000:2.0:1787829725.570840:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c68f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572631296:192.168.203.129@tcp:38:kworker.0 00000800:00000100:2.0:1787829725.570850:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787829725.570856:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829725/real 1787829725] req@ffff8cb999c68f00 x1874672572631296/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829750 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829725.570864:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c68f00 x1874672572631296/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829750 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829725.570879:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829725.570881:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787829725.570884:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c68f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572631296:192.168.203.129@tcp:38:kworker.0 00010000:00010000:2.0:1787829726.402482:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829727.426441:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829728.450440:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829729.474453:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829730.498460:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787829730.690395:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829730.690405:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787829730.690410:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829730.690420:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 2682 00000100:00080000:2.0:1787829730.690432:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829730.690438:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:2.0:1787829730.690481:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8e1c0 x1874672572631424/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829730.690511:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829730.690515:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787829730.690516:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829730.690519:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 2682 00000100:00080000:2.0:1787829730.690522:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829730.690537:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:2.0:1787829730.690549:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787829730.690561:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8e1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572631424:192.168.203.129@tcp:250:kworker.0 00000100:00100000:2.0:1787829730.690566:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8da40 x1874672572631552/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829730.690599:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829730.690602:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1787829730.690604:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829730.690608:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 2682 00000100:00080000:2.0:1787829730.690612:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829730.690616:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:2.0:1787829730.690620:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787829730.690632:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8de00 x1874672572631680/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1787829730.690636:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787829730.690696:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829730/real 1787829730] req@ffff8cb998a8e1c0 x1874672572631424/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829755 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829730.690719:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8e1c0 x1874672572631424/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829755 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829730.690733:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829730.690738:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787829730.690742:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8e1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572631424:192.168.203.129@tcp:250:kworker.0 00000100:00100000:0.0:1787829730.690770:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8da40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572631552:192.168.203.129@tcp:38:kworker.0 00000800:00000100:0.0:1787829730.690790:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787829730.690804:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829730/real 1787829730] req@ffff8cb998a8da40 x1874672572631552/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829755 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829730.690818:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8da40 x1874672572631552/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829755 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829730.690828:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829730.690832:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787829730.690836:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8da40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572631552:192.168.203.129@tcp:38:kworker.0 00000100:00100000:0.0:1787829730.690843:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8de00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572631680:192.168.203.129@tcp:38:kworker.0 00000800:00000100:0.0:1787829730.690869:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787829730.690878:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829730/real 1787829730] req@ffff8cb998a8de00 x1874672572631680/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829755 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829730.690886:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8de00 x1874672572631680/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829755 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829730.690892:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829730.690894:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787829730.690897:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8de00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572631680:192.168.203.129@tcp:38:kworker.0 00010000:00010000:2.0:1787829731.522403:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829732.546438:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829733.570429:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829734.594440:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787829735.618461:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787829735.810340:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829735.810351:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787829735.810356:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829735.810366:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 2687 00000100:00080000:2.0:1787829735.810378:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829735.810384:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:2.0:1787829735.810426:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8c3c0 x1874672572631808/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829735.810453:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829735.810455:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787829735.810474:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829735.810478:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 2687 00000100:00080000:2.0:1787829735.810481:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829735.810485:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:2.0:1787829735.810511:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787829735.810523:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8d2c0 x1874672572631936/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787829735.810534:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8c3c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572631808:192.168.203.129@tcp:250:kworker.0 00000100:00080000:2.0:1787829735.810534:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829735.810541:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1787829735.810545:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829735.810549:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 2687 00000100:00080000:2.0:1787829735.810552:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829735.810554:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:2.0:1787829735.810558:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787829735.810569:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8de00 x1874672572632064/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1787829735.810620:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787829735.810691:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829735/real 1787829735] req@ffff8cb998a8c3c0 x1874672572631808/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829760 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829735.810713:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8c3c0 x1874672572631808/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829760 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829735.810726:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829735.810730:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787829735.810735:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8c3c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572631808:192.168.203.129@tcp:250:kworker.0 00000100:00100000:0.0:1787829735.810764:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8d2c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572631936:192.168.203.129@tcp:38:kworker.0 00000800:00000100:0.0:1787829735.810789:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787829735.810804:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829735/real 1787829735] req@ffff8cb998a8d2c0 x1874672572631936/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829760 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829735.810831:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8d2c0 x1874672572631936/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829760 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829735.810839:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829735.810843:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787829735.810846:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8d2c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572631936:192.168.203.129@tcp:38:kworker.0 00000100:00100000:0.0:1787829735.810852:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8de00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572632064:192.168.203.129@tcp:38:kworker.0 00000800:00000100:0.0:1787829735.810875:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787829735.810888:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829735/real 1787829735] req@ffff8cb998a8de00 x1874672572632064/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829760 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829735.810900:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8de00 x1874672572632064/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829760 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829735.810909:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829735.810912:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787829735.810916:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8de00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572632064:192.168.203.129@tcp:38:kworker.0 00010000:00010000:2.0:1787829736.642422:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829737.666366:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787829738.690443:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829739.714400:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829740.738471:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787829740.930409:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829740.930420:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787829740.930424:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829740.930436:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 2693 00000100:00080000:1.0:1787829740.930449:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829740.930504:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:1.0:1787829740.930542:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f38b40 x1874672572632192/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829740.930570:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829740.930573:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787829740.930574:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829740.930577:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 2693 00000100:00080000:1.0:1787829740.930580:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829740.930582:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:1.0:1787829740.930591:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787829740.930599:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f39a40 x1874672572632320/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829740.930607:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829740.930610:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1787829740.930611:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829740.930614:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 2693 00000100:00080000:1.0:1787829740.930616:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829740.930619:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:1.0:1787829740.930621:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787829740.930629:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f38b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572632192:192.168.203.129@tcp:250:kworker.0 00000100:00100000:1.0:1787829740.930630:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f39e00 x1874672572632448/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1787829740.930702:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787829740.930752:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829740/real 1787829740] req@ffff8cb999f38b40 x1874672572632192/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829765 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829740.930767:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f38b40 x1874672572632192/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829765 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829740.930787:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829740.930791:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787829740.930794:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f38b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572632192:192.168.203.129@tcp:250:kworker.0 00000100:00100000:0.0:1787829740.930800:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f39a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572632320:192.168.203.129@tcp:38:kworker.0 00000800:00000100:0.0:1787829740.930826:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787829740.930836:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829740/real 1787829740] req@ffff8cb999f39a40 x1874672572632320/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829765 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829740.930844:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f39a40 x1874672572632320/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829765 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829740.930850:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829740.930852:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787829740.930854:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f39a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572632320:192.168.203.129@tcp:38:kworker.0 00000100:00100000:0.0:1787829740.930879:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f39e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572632448:192.168.203.129@tcp:38:kworker.0 00000800:00000100:0.0:1787829740.930891:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787829740.930897:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829740/real 1787829740] req@ffff8cb999f39e00 x1874672572632448/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829765 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829740.930907:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f39e00 x1874672572632448/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829765 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829740.930916:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829740.930919:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787829740.930922:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f39e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572632448:192.168.203.129@tcp:38:kworker.0 00010000:00010000:1.0:1787829741.762551:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829742.786412:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829743.810406:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787829744.834456:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829745.858463:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787829746.050369:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829746.050382:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787829746.050388:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829746.050402:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 2698 00000100:00080000:1.0:1787829746.050415:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829746.050438:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:1.0:1787829746.050498:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f3b840 x1874672572632576/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829746.050529:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829746.050533:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787829746.050535:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829746.050538:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 2698 00000100:00080000:1.0:1787829746.050542:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829746.050545:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:1.0:1787829746.050569:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787829746.050602:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f3b840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572632576:192.168.203.129@tcp:250:kworker.0 00000100:00100000:1.0:1787829746.050603:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f39e00 x1874672572632704/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829746.050619:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829746.050622:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1787829746.050624:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829746.050628:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 2698 00000100:00080000:1.0:1787829746.050632:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829746.050634:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:1.0:1787829746.050638:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787829746.050657:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f39a40 x1874672572632832/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1787829746.050683:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787829746.050737:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829746/real 1787829746] req@ffff8cb999f3b840 x1874672572632576/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829771 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829746.050771:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f3b840 x1874672572632576/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829771 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829746.050782:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829746.050786:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787829746.050791:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f3b840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572632576:192.168.203.129@tcp:250:kworker.0 00000100:00100000:2.0:1787829746.050819:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f39e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572632704:192.168.203.129@tcp:38:kworker.0 00000800:00000100:2.0:1787829746.050838:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787829746.050851:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829746/real 1787829746] req@ffff8cb999f39e00 x1874672572632704/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829771 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829746.050862:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f39e00 x1874672572632704/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829771 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829746.050868:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829746.050871:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787829746.050873:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f39e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572632704:192.168.203.129@tcp:38:kworker.0 00000100:00100000:2.0:1787829746.050877:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f39a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572632832:192.168.203.129@tcp:38:kworker.0 00000800:00000100:2.0:1787829746.050891:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787829746.050899:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829746/real 1787829746] req@ffff8cb999f39a40 x1874672572632832/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829771 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829746.050907:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f39a40 x1874672572632832/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829771 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829746.050924:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829746.050927:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787829746.050930:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f39a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572632832:192.168.203.129@tcp:38:kworker.0 00010000:00010000:2.0:1787829746.882446:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787829747.906453:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829748.930442:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829749.954431:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787829750.978502:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787829751.170398:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829751.170413:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787829751.170419:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829751.170430:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 2703 00000100:00080000:2.0:1787829751.170444:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829751.170452:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:2.0:1787829751.170501:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8da40 x1874672572632960/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829751.170533:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829751.170538:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787829751.170541:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829751.170546:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 2703 00000100:00080000:2.0:1787829751.170552:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829751.170559:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:2.0:1787829751.170574:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787829751.170581:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8da40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572632960:192.168.203.129@tcp:250:kworker.0 00000100:00100000:2.0:1787829751.170597:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8e1c0 x1874672572633088/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829751.170640:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829751.170643:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1787829751.170645:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829751.170650:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 2703 00000100:00080000:2.0:1787829751.170656:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829751.170660:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000800:00000100:0.0:1787829751.170666:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000008:00100000:2.0:1787829751.170666:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787829751.170686:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8fc00 x1874672572633216/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787829751.170733:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829751/real 1787829751] req@ffff8cb998a8da40 x1874672572632960/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829776 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829751.170760:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8da40 x1874672572632960/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829776 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829751.170776:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829751.170781:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787829751.170786:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8da40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572632960:192.168.203.129@tcp:250:kworker.0 00000100:00100000:0.0:1787829751.170815:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8e1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572633088:192.168.203.129@tcp:38:kworker.0 00000800:00000100:0.0:1787829751.170841:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787829751.170854:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829751/real 1787829751] req@ffff8cb998a8e1c0 x1874672572633088/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829776 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829751.170865:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8e1c0 x1874672572633088/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829776 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829751.170872:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829751.170875:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787829751.170878:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8e1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572633088:192.168.203.129@tcp:38:kworker.0 00000100:00100000:0.0:1787829751.170883:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8fc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572633216:192.168.203.129@tcp:38:kworker.0 00000800:00000100:0.0:1787829751.170897:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787829751.170905:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829751/real 1787829751] req@ffff8cb998a8fc00 x1874672572633216/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829776 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829751.170926:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8fc00 x1874672572633216/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829776 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829751.170933:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829751.170936:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787829751.170939:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8fc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572633216:192.168.203.129@tcp:38:kworker.0 00010000:00010000:2.0:1787829752.002408:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829753.026441:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787829754.050481:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829755.074399:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829756.098428:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787829756.290388:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829756.290397:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787829756.290402:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829756.290413:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 2708 00000100:00080000:0.0:1787829756.290425:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829756.290431:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:0.0:1787829756.290477:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c6b840 x1874672572633344/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829756.290507:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829756.290510:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787829756.290512:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829756.290515:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 2708 00000100:00080000:0.0:1787829756.290517:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829756.290520:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:0.0:1787829756.290529:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787829756.290538:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c69a40 x1874672572633472/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787829756.290543:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c6b840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572633344:192.168.203.129@tcp:250:kworker.0 00000100:00080000:0.0:1787829756.290571:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829756.290574:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1787829756.290576:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829756.290580:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 2708 00000100:00080000:0.0:1787829756.290583:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829756.290586:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:0.0:1787829756.290590:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787829756.290600:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c68f00 x1874672572633600/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1787829756.290616:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787829756.290668:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829756/real 1787829756] req@ffff8cb999c6b840 x1874672572633344/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829781 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829756.290683:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c6b840 x1874672572633344/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829781 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829756.290693:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829756.290697:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787829756.290700:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c6b840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572633344:192.168.203.129@tcp:250:kworker.0 00000100:00100000:2.0:1787829756.290724:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c69a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572633472:192.168.203.129@tcp:38:kworker.0 00000800:00000100:2.0:1787829756.290748:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787829756.290757:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829756/real 1787829756] req@ffff8cb999c69a40 x1874672572633472/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829781 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829756.290766:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c69a40 x1874672572633472/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829781 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829756.290771:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829756.290774:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787829756.290776:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c69a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572633472:192.168.203.129@tcp:38:kworker.0 00000100:00100000:2.0:1787829756.290779:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c68f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572633600:192.168.203.129@tcp:38:kworker.0 00000800:00000100:2.0:1787829756.290793:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787829756.290804:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829756/real 1787829756] req@ffff8cb999c68f00 x1874672572633600/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829781 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829756.290817:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c68f00 x1874672572633600/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829781 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829756.290826:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829756.290830:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787829756.290834:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c68f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572633600:192.168.203.129@tcp:38:kworker.0 00010000:00010000:1.0:1787829757.122424:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829758.146427:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829759.170418:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787829760.194406:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829761.218545:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787829761.410450:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829761.410460:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787829761.410487:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829761.410498:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 2713 00000100:00080000:2.0:1787829761.410510:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829761.410517:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:2.0:1787829761.410555:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8f0c0 x1874672572633728/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829761.410585:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829761.410588:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787829761.410589:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829761.410592:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 2713 00000100:00080000:2.0:1787829761.410595:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829761.410599:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:2.0:1787829761.410609:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787829761.410620:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8c780 x1874672572633856/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787829761.410622:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8f0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572633728:192.168.203.129@tcp:250:kworker.0 00000100:00080000:2.0:1787829761.410637:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829761.410640:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1787829761.410641:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829761.410644:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 2713 00000100:00080000:2.0:1787829761.410646:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829761.410648:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:2.0:1787829761.410652:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787829761.410659:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8cf00 x1874672572633984/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1787829761.410697:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787829761.410738:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829761/real 1787829761] req@ffff8cb998a8f0c0 x1874672572633728/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829786 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829761.410753:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8f0c0 x1874672572633728/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829786 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829761.410763:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829761.410766:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787829761.410770:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8f0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572633728:192.168.203.129@tcp:250:kworker.0 00000100:00100000:0.0:1787829761.410793:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8c780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572633856:192.168.203.129@tcp:38:kworker.0 00000800:00000100:0.0:1787829761.410808:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787829761.410817:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829761/real 1787829761] req@ffff8cb998a8c780 x1874672572633856/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829786 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829761.410843:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8c780 x1874672572633856/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829786 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829761.410849:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829761.410852:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787829761.410854:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8c780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572633856:192.168.203.129@tcp:38:kworker.0 00000100:00100000:0.0:1787829761.410859:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8cf00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572633984:192.168.203.129@tcp:38:kworker.0 00000800:00000100:0.0:1787829761.410875:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787829761.410882:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829761/real 1787829761] req@ffff8cb998a8cf00 x1874672572633984/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829786 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829761.410890:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8cf00 x1874672572633984/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829786 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829761.410896:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829761.410897:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787829761.410899:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8cf00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572633984:192.168.203.129@tcp:38:kworker.0 00010000:00010000:1.0:1787829762.242448:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787829763.266423:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829764.290434:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787829765.314440:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787829766.338433:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787829766.530376:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829766.530384:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787829766.530388:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829766.530396:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 2718 00000100:00080000:0.0:1787829766.530405:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829766.530411:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:0.0:1787829766.530443:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c6bc00 x1874672572634112/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829766.530477:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829766.530480:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787829766.530481:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829766.530484:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 2718 00000100:00080000:0.0:1787829766.530487:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829766.530490:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:0.0:1787829766.530498:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787829766.530508:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c69e00 x1874672572634240/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829766.530515:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829766.530518:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1787829766.530519:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829766.530522:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 2718 00000100:00080000:0.0:1787829766.530525:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829766.530530:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:2.0:1787829766.530530:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c6bc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572634112:192.168.203.129@tcp:250:kworker.0 00000008:00100000:0.0:1787829766.530536:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787829766.530553:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c6b480 x1874672572634368/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1787829766.530600:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787829766.530650:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829766/real 1787829766] req@ffff8cb999c6bc00 x1874672572634112/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829791 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829766.530670:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c6bc00 x1874672572634112/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829791 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829766.530682:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829766.530687:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787829766.530691:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c6bc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572634112:192.168.203.129@tcp:250:kworker.0 00000100:00100000:2.0:1787829766.530699:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c69e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572634240:192.168.203.129@tcp:38:kworker.0 00000800:00000100:2.0:1787829766.530723:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787829766.530737:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829766/real 1787829766] req@ffff8cb999c69e00 x1874672572634240/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829791 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829766.530750:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c69e00 x1874672572634240/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829791 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829766.530759:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829766.530763:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787829766.530767:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c69e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572634240:192.168.203.129@tcp:38:kworker.0 00000100:00100000:2.0:1787829766.530794:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c6b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572634368:192.168.203.129@tcp:38:kworker.0 00000800:00000100:2.0:1787829766.530828:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787829766.530840:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829766/real 1787829766] req@ffff8cb999c6b480 x1874672572634368/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829791 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829766.530854:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c6b480 x1874672572634368/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829791 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829766.530863:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829766.530867:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787829766.530871:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c6b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572634368:192.168.203.129@tcp:38:kworker.0 00010000:00010000:2.0:1787829767.362438:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787829768.386472:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787829769.410467:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829770.434396:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787829771.458447:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787829771.650389:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829771.650397:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787829771.650400:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829771.650408:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 2723 00000100:00080000:2.0:1787829771.650419:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829771.650425:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:2.0:1787829771.650456:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8e580 x1874672572634496/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829771.650478:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829771.650481:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787829771.650483:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829771.650485:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 2723 00000100:00080000:2.0:1787829771.650488:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829771.650504:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:2.0:1787829771.650512:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787829771.650524:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8e580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572634496:192.168.203.129@tcp:250:kworker.0 00000100:00100000:2.0:1787829771.650524:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8d680 x1874672572634624/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829771.650541:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829771.650543:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1787829771.650545:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829771.650547:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 2723 00000100:00080000:2.0:1787829771.650551:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829771.650553:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:2.0:1787829771.650556:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787829771.650565:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8f480 x1874672572634752/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1787829771.650609:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787829771.650653:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829771/real 1787829771] req@ffff8cb998a8e580 x1874672572634496/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829796 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829771.650697:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8e580 x1874672572634496/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829796 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829771.650706:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829771.650709:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787829771.650713:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8e580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572634496:192.168.203.129@tcp:250:kworker.0 00000100:00100000:0.0:1787829771.650734:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8d680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572634624:192.168.203.129@tcp:38:kworker.0 00000800:00000100:0.0:1787829771.650753:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787829771.650763:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829771/real 1787829771] req@ffff8cb998a8d680 x1874672572634624/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829796 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829771.650772:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8d680 x1874672572634624/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829796 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829771.650778:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829771.650780:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787829771.650783:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8d680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572634624:192.168.203.129@tcp:38:kworker.0 00000100:00100000:0.0:1787829771.650786:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572634752:192.168.203.129@tcp:38:kworker.0 00000800:00000100:0.0:1787829771.650797:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787829771.650804:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829771/real 1787829771] req@ffff8cb998a8f480 x1874672572634752/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829796 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829771.650812:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8f480 x1874672572634752/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829796 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829771.650826:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829771.650830:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787829771.650832:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572634752:192.168.203.129@tcp:38:kworker.0 00010000:00010000:1.0:1787829772.482470:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829773.506475:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787829774.530440:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829775.555440:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829776.578396:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787829776.770399:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829776.770410:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787829776.770414:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829776.770425:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 2728 00000100:00080000:1.0:1787829776.770437:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829776.770444:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:1.0:1787829776.770477:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f38b40 x1874672572634880/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829776.770504:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829776.770508:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787829776.770509:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829776.770513:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 2728 00000100:00080000:1.0:1787829776.770527:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829776.770533:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:1.0:1787829776.770542:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787829776.770557:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f38b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572634880:192.168.203.129@tcp:250:kworker.0 00000100:00100000:1.0:1787829776.770562:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f3b480 x1874672572635008/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829776.770573:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829776.770577:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1787829776.770579:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829776.770583:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 2728 00000100:00080000:1.0:1787829776.770587:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829776.770591:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:1.0:1787829776.770595:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787829776.770606:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f3b0c0 x1874672572635136/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787829776.770679:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f3b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572635008:192.168.203.129@tcp:38:kworker.0 00000100:00100000:0.0:1787829776.770706:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f3b0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572635136:192.168.203.129@tcp:38:kworker.0 00010000:00010000:0.0:1787829777.602442:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829778.626468:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829779.650408:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:3.0:1787829779.842583:0:2381:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.129:988 00000400:00000100:3.0:1787829779.842665:0:2381:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.129@tcp at host 192.168.203.129 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:3.0:1787829779.842686:0:2381:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.129@tcp 00000800:00000100:3.0:1787829779.842745:0:2381:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.129@tcp 00000800:00000100:3.0:1787829779.842772:0:2381:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.129@tcp 00000100:00100000:2.0:1787829779.842803:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829776/real 1787829779] req@ffff8cb999f3b480 x1874672572635008/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829801 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829779.842831:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f3b480 x1874672572635008/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829801 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829779.842845:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829779.842850:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787829779.842879:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f3b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572635008:192.168.203.129@tcp:38:kworker.0 00000100:00100000:2.0:1787829779.842887:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829776/real 1787829779] req@ffff8cb999f3b0c0 x1874672572635136/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829801 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829779.842914:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f3b0c0 x1874672572635136/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829801 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829779.842923:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829779.842927:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787829779.842931:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f3b0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572635136:192.168.203.129@tcp:38:kworker.0 00000100:00100000:2.0:1787829779.842938:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829776/real 1787829779] req@ffff8cb999f38b40 x1874672572634880/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829801 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829779.842950:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f38b40 x1874672572634880/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829801 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829779.842963:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829779.842967:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787829779.842980:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f38b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572634880:192.168.203.129@tcp:250:kworker.0 00010000:00010000:0.0:1787829780.674411:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829781.698426:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787829781.890378:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829781.890387:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787829781.890391:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829781.890402:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 2734 00000100:00080000:1.0:1787829781.890413:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829781.890421:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:1.0:1787829781.890453:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f38f00 x1874672572635264/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829781.890477:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829781.890493:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787829781.890495:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829781.890501:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 2734 00000100:00080000:1.0:1787829781.890507:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829781.890513:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:1.0:1787829781.890523:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787829781.890523:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f38f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572635264:192.168.203.129@tcp:250:kworker.0 00000100:00100000:1.0:1787829781.890543:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f392c0 x1874672572635392/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829781.890556:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829781.890560:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1787829781.890563:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829781.890567:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 2734 00000100:00080000:1.0:1787829781.890571:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829781.890575:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:1.0:1787829781.890580:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000800:00000100:2.0:1787829781.890580:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787829781.890594:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f3a580 x1874672572635520/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787829781.890629:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829781/real 1787829781] req@ffff8cb999f38f00 x1874672572635264/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829806 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829781.890648:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f38f00 x1874672572635264/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829806 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829781.890677:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829781.890682:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787829781.890686:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f38f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572635264:192.168.203.129@tcp:250:kworker.0 00000100:00100000:2.0:1787829781.890717:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f392c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572635392:192.168.203.129@tcp:38:kworker.0 00000800:00000100:2.0:1787829781.890767:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787829781.890781:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829781/real 1787829781] req@ffff8cb999f392c0 x1874672572635392/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829806 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829781.890793:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f392c0 x1874672572635392/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829806 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829781.890800:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829781.890804:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787829781.890807:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f392c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572635392:192.168.203.129@tcp:38:kworker.0 00000100:00100000:2.0:1787829781.890813:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f3a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572635520:192.168.203.129@tcp:38:kworker.0 00000800:00000100:2.0:1787829781.890832:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787829781.890843:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829781/real 1787829781] req@ffff8cb999f3a580 x1874672572635520/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829806 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829781.890854:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f3a580 x1874672572635520/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829806 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829781.890862:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829781.890865:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787829781.890869:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f3a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572635520:192.168.203.129@tcp:38:kworker.0 00010000:00010000:2.0:1787829782.722450:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787829783.746378:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829784.770705:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829785.794434:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787829786.818446:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787829787.010387:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829787.010398:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787829787.010402:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829787.010413:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 2739 00000100:00080000:0.0:1787829787.010424:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829787.010431:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:0.0:1787829787.010473:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c692c0 x1874672572635648/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829787.010497:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829787.010500:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787829787.010501:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829787.010504:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 2739 00000100:00080000:0.0:1787829787.010530:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829787.010535:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:0.0:1787829787.010551:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787829787.010566:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c69680 x1874672572635776/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829787.010578:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1787829787.010584:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c692c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572635648:192.168.203.129@tcp:250:kworker.0 00000100:00080000:0.0:1787829787.010585:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1787829787.010590:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829787.010597:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 2739 00000100:00080000:0.0:1787829787.010601:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829787.010605:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:0.0:1787829787.010610:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787829787.010625:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c6a1c0 x1874672572635904/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1787829787.010669:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787829787.010719:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829787/real 1787829787] req@ffff8cb999c692c0 x1874672572635648/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829812 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829787.010733:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c692c0 x1874672572635648/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829812 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829787.010744:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829787.010747:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787829787.010751:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c692c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572635648:192.168.203.129@tcp:250:kworker.0 00000100:00100000:2.0:1787829787.010779:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c69680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572635776:192.168.203.129@tcp:38:kworker.0 00000800:00000100:2.0:1787829787.010795:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787829787.010803:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829787/real 1787829787] req@ffff8cb999c69680 x1874672572635776/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829812 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829787.010811:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c69680 x1874672572635776/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829812 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829787.010816:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829787.010819:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787829787.010821:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c69680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572635776:192.168.203.129@tcp:38:kworker.0 00000100:00100000:2.0:1787829787.010824:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c6a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572635904:192.168.203.129@tcp:38:kworker.0 00000800:00000100:2.0:1787829787.010846:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787829787.010853:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829787/real 1787829787] req@ffff8cb999c6a1c0 x1874672572635904/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829812 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829787.010860:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c6a1c0 x1874672572635904/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829812 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829787.010865:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829787.010867:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787829787.010869:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c6a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572635904:192.168.203.129@tcp:38:kworker.0 00010000:00010000:2.0:1787829787.842637:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829788.866898:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787829789.890439:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829790.914483:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829791.938448:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787829792.130333:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829792.130343:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787829792.130348:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829792.130358:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 2744 00000100:00080000:0.0:1787829792.130371:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829792.130378:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:0.0:1787829792.130420:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c6a580 x1874672572636032/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829792.130447:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829792.130486:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787829792.130489:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829792.130493:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 2744 00000100:00080000:0.0:1787829792.130498:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829792.130502:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:0.0:1787829792.130514:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787829792.130523:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c6a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572636032:192.168.203.129@tcp:250:kworker.0 00000100:00100000:0.0:1787829792.130530:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c68000 x1874672572636160/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829792.130545:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829792.130548:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1787829792.130550:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829792.130553:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 2744 00000100:00080000:0.0:1787829792.130556:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829792.130559:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:0.0:1787829792.130563:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787829792.130574:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c6a1c0 x1874672572636288/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1787829792.130640:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787829792.130730:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829792/real 1787829792] req@ffff8cb999c6a580 x1874672572636032/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829817 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829792.130775:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c6a580 x1874672572636032/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829817 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829792.130785:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829792.130788:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787829792.130792:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c6a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572636032:192.168.203.129@tcp:250:kworker.0 00000100:00100000:2.0:1787829792.130819:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c68000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572636160:192.168.203.129@tcp:38:kworker.0 00000800:00000100:2.0:1787829792.130868:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787829792.130882:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829792/real 1787829792] req@ffff8cb999c68000 x1874672572636160/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829817 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829792.130894:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c68000 x1874672572636160/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829817 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829792.130902:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829792.130905:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787829792.130908:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c68000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572636160:192.168.203.129@tcp:38:kworker.0 00000100:00100000:2.0:1787829792.130913:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c6a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572636288:192.168.203.129@tcp:38:kworker.0 00000800:00000100:2.0:1787829792.130928:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787829792.130937:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829792/real 1787829792] req@ffff8cb999c6a1c0 x1874672572636288/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829817 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829792.130947:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c6a1c0 x1874672572636288/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829817 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829792.130953:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829792.130956:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787829792.130958:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c6a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572636288:192.168.203.129@tcp:38:kworker.0 00010000:00010000:0.0:1787829792.962387:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829793.986438:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829795.010472:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787829796.034401:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829797.058415:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787829797.250363:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829797.250373:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787829797.250378:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829797.250389:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 2749 00000100:00080000:1.0:1787829797.250401:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829797.250408:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:1.0:1787829797.250450:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f383c0 x1874672572636416/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829797.250500:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829797.250504:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787829797.250505:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829797.250508:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 2749 00000100:00080000:1.0:1787829797.250512:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829797.250515:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:1.0:1787829797.250525:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787829797.250537:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f3a580 x1874672572636544/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829797.250546:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829797.250550:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1787829797.250553:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1787829797.250553:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f383c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572636416:192.168.203.129@tcp:250:kworker.0 00000100:00080000:1.0:1787829797.250561:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 2749 00000100:00080000:1.0:1787829797.250565:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829797.250567:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:1.0:1787829797.250570:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787829797.250586:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f392c0 x1874672572636672/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1787829797.250639:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787829797.250696:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829797/real 1787829797] req@ffff8cb999f383c0 x1874672572636416/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829822 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829797.250712:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f383c0 x1874672572636416/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829822 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829797.250721:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829797.250725:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787829797.250728:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f383c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572636416:192.168.203.129@tcp:250:kworker.0 00000100:00100000:2.0:1787829797.250760:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f3a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572636544:192.168.203.129@tcp:38:kworker.0 00000800:00000100:2.0:1787829797.250783:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787829797.250794:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829797/real 1787829797] req@ffff8cb999f3a580 x1874672572636544/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829822 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829797.250802:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f3a580 x1874672572636544/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829822 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829797.250808:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829797.250811:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787829797.250813:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f3a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572636544:192.168.203.129@tcp:38:kworker.0 00000100:00100000:2.0:1787829797.250817:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f392c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572636672:192.168.203.129@tcp:38:kworker.0 00000800:00000100:2.0:1787829797.250829:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787829797.250835:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829797/real 1787829797] req@ffff8cb999f392c0 x1874672572636672/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829822 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829797.250843:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f392c0 x1874672572636672/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829822 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829797.250859:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829797.250861:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787829797.250863:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f392c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572636672:192.168.203.129@tcp:38:kworker.0 00010000:00010000:2.0:1787829798.082503:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787829799.106426:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829800.130420:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829801.154393:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829802.178482:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787829802.370419:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829802.370433:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787829802.370439:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829802.370453:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 2754 00000100:00080000:2.0:1787829802.370467:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829802.370477:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:2.0:1787829802.370527:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8c000 x1874672572636800/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829802.370562:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829802.370567:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787829802.370576:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829802.370581:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 2754 00000100:00080000:2.0:1787829802.370587:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829802.370592:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:0.0:1787829802.370605:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572636800:192.168.203.129@tcp:250:kworker.0 00000008:00100000:2.0:1787829802.370605:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787829802.370624:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8cb40 x1874672572636928/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829802.370656:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829802.370660:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1787829802.370662:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829802.370667:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 2754 00000100:00080000:2.0:1787829802.370672:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000800:00000100:0.0:1787829802.370677:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00080000:2.0:1787829802.370680:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:2.0:1787829802.370688:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787829802.370713:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8ed00 x1874672572637056/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787829802.370739:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829802/real 1787829802] req@ffff8cb998a8c000 x1874672572636800/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829827 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829802.370779:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8c000 x1874672572636800/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829827 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829802.370788:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829802.370792:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787829802.370796:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572636800:192.168.203.129@tcp:250:kworker.0 00000100:00100000:0.0:1787829802.370819:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8cb40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572636928:192.168.203.129@tcp:38:kworker.0 00000800:00000100:0.0:1787829802.370840:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787829802.370850:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829802/real 1787829802] req@ffff8cb998a8cb40 x1874672572636928/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829827 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829802.370858:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8cb40 x1874672572636928/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829827 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829802.370864:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829802.370867:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787829802.370869:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8cb40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572636928:192.168.203.129@tcp:38:kworker.0 00000100:00100000:0.0:1787829802.370872:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8ed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572637056:192.168.203.129@tcp:38:kworker.0 00000800:00000100:0.0:1787829802.370883:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787829802.370889:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829802/real 1787829802] req@ffff8cb998a8ed00 x1874672572637056/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829827 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829802.370897:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8ed00 x1874672572637056/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829827 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829802.370912:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829802.370914:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787829802.370916:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8ed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572637056:192.168.203.129@tcp:38:kworker.0 00010000:00010000:2.0:1787829803.202432:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829804.226446:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829805.250389:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829806.274473:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829807.298470:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787829807.490371:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829807.490386:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787829807.490392:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829807.490405:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 2759 00000100:00080000:2.0:1787829807.490420:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829807.490429:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:2.0:1787829807.490480:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8f840 x1874672572637184/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829807.490519:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829807.490529:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787829807.490550:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829807.490556:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 2759 00000100:00080000:2.0:1787829807.490561:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829807.490567:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:0.0:1787829807.490580:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8f840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572637184:192.168.203.129@tcp:250:kworker.0 00000008:00100000:2.0:1787829807.490581:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787829807.490602:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8e940 x1874672572637312/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829807.490616:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829807.490620:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1787829807.490623:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829807.490627:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 2759 00000100:00080000:2.0:1787829807.490632:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829807.490636:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:2.0:1787829807.490643:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000800:00000100:0.0:1787829807.490652:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787829807.490661:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8ed00 x1874672572637440/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787829807.490699:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829807/real 1787829807] req@ffff8cb998a8f840 x1874672572637184/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829832 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829807.490720:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8f840 x1874672572637184/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829832 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829807.490733:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829807.490738:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787829807.490742:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8f840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572637184:192.168.203.129@tcp:250:kworker.0 00000100:00100000:0.0:1787829807.490771:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8e940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572637312:192.168.203.129@tcp:38:kworker.0 00000800:00000100:0.0:1787829807.490792:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787829807.490803:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829807/real 1787829807] req@ffff8cb998a8e940 x1874672572637312/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829832 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829807.490812:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8e940 x1874672572637312/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829832 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829807.490818:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829807.490821:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787829807.490823:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8e940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572637312:192.168.203.129@tcp:38:kworker.0 00000100:00100000:0.0:1787829807.490835:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8ed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572637440:192.168.203.129@tcp:38:kworker.0 00000800:00000100:0.0:1787829807.490850:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787829807.490857:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829807/real 1787829807] req@ffff8cb998a8ed00 x1874672572637440/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829832 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829807.490865:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8ed00 x1874672572637440/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829832 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829807.490871:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829807.490873:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787829807.490876:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8ed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572637440:192.168.203.129@tcp:38:kworker.0 00010000:00010000:2.0:1787829808.322480:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829809.346442:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829810.370450:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829811.394402:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829812.418423:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787829812.610408:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829812.610420:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787829812.610424:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829812.610434:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 2764 00000100:00080000:0.0:1787829812.610447:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829812.610454:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:0.0:1787829812.610506:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c69680 x1874672572637568/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829812.610533:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829812.610537:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787829812.610538:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829812.610541:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 2764 00000100:00080000:0.0:1787829812.610544:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829812.610565:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:2.0:1787829812.610571:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c69680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572637568:192.168.203.129@tcp:250:kworker.0 00000008:00100000:0.0:1787829812.610578:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787829812.610591:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c692c0 x1874672572637696/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829812.610600:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829812.610603:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1787829812.610605:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829812.610610:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 2764 00000100:00080000:0.0:1787829812.610615:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829812.610620:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:0.0:1787829812.610625:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787829812.610641:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c6b480 x1874672572637824/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1787829812.610651:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787829812.610714:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829812/real 1787829812] req@ffff8cb999c69680 x1874672572637568/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829837 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829812.610742:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c69680 x1874672572637568/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829837 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829812.610756:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829812.610761:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787829812.610768:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c69680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572637568:192.168.203.129@tcp:250:kworker.0 00000100:00100000:2.0:1787829812.610798:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c692c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572637696:192.168.203.129@tcp:38:kworker.0 00000800:00000100:2.0:1787829812.610819:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787829812.610829:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829812/real 1787829812] req@ffff8cb999c692c0 x1874672572637696/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829837 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829812.610838:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c692c0 x1874672572637696/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829837 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829812.610844:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829812.610846:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787829812.610849:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c692c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572637696:192.168.203.129@tcp:38:kworker.0 00000100:00100000:2.0:1787829812.610852:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c6b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572637824:192.168.203.129@tcp:38:kworker.0 00000800:00000100:2.0:1787829812.610864:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787829812.610870:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829812/real 1787829812] req@ffff8cb999c6b480 x1874672572637824/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829837 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829812.610878:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c6b480 x1874672572637824/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829837 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829812.610892:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829812.610895:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787829812.610897:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c6b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572637824:192.168.203.129@tcp:38:kworker.0 00010000:00010000:2.0:1787829813.442441:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829814.466443:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829815.490429:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829816.514435:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829817.538437:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787829817.730354:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829817.730369:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787829817.730374:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829817.730386:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 2769 00000100:00080000:0.0:1787829817.730401:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829817.730411:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:0.0:1787829817.730459:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c69e00 x1874672572637952/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829817.730488:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829817.730494:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787829817.730503:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829817.730508:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 2769 00000100:00080000:0.0:1787829817.730513:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829817.730518:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:2.0:1787829817.730525:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c69e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572637952:192.168.203.129@tcp:250:kworker.0 00000008:00100000:0.0:1787829817.730542:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787829817.730562:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c6bc00 x1874672572638080/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829817.730576:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829817.730580:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1787829817.730583:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829817.730588:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 2769 00000100:00080000:0.0:1787829817.730596:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829817.730601:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:0.0:1787829817.730607:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000800:00000100:2.0:1787829817.730610:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787829817.730625:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c68f00 x1874672572638208/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787829817.730662:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829817/real 1787829817] req@ffff8cb999c69e00 x1874672572637952/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829842 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829817.730698:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c69e00 x1874672572637952/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829842 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829817.730712:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829817.730718:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787829817.730723:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c69e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572637952:192.168.203.129@tcp:250:kworker.0 00000100:00100000:2.0:1787829817.730753:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c6bc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572638080:192.168.203.129@tcp:38:kworker.0 00000800:00000100:2.0:1787829817.730780:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787829817.730797:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829817/real 1787829817] req@ffff8cb999c6bc00 x1874672572638080/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829842 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829817.730812:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c6bc00 x1874672572638080/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829842 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829817.730829:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829817.730834:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787829817.730838:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c6bc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572638080:192.168.203.129@tcp:38:kworker.0 00000100:00100000:2.0:1787829817.730845:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c68f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572638208:192.168.203.129@tcp:38:kworker.0 00000800:00000100:2.0:1787829817.730867:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787829817.730878:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829817/real 1787829817] req@ffff8cb999c68f00 x1874672572638208/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829842 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829817.730891:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c68f00 x1874672572638208/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829842 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829817.730901:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829817.730906:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787829817.730910:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c68f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572638208:192.168.203.129@tcp:38:kworker.0 00010000:00010000:2.0:1787829818.562441:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829819.586437:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829820.610397:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829821.634484:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829822.658436:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787829822.850410:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829822.850422:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787829822.850428:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829822.850440:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 2775 00000100:00080000:0.0:1787829822.850453:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829822.850461:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:0.0:1787829822.850507:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c69a40 x1874672572638336/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829822.850557:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829822.850561:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787829822.850563:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829822.850567:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 2775 00000100:00080000:0.0:1787829822.850571:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829822.850576:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:0.0:1787829822.850588:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787829822.850597:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c69a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572638336:192.168.203.129@tcp:250:kworker.0 00000100:00100000:0.0:1787829822.850602:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c6b840 x1874672572638464/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829822.850614:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829822.850618:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1787829822.850620:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829822.850623:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 2775 00000100:00080000:0.0:1787829822.850627:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829822.850631:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:0.0:1787829822.850636:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787829822.850646:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c68f00 x1874672572638592/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1787829822.850664:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787829822.850715:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829822/real 1787829822] req@ffff8cb999c69a40 x1874672572638336/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829847 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829822.850734:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c69a40 x1874672572638336/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829847 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829822.850752:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829822.850757:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787829822.850761:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c69a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572638336:192.168.203.129@tcp:250:kworker.0 00000100:00100000:2.0:1787829822.850798:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c6b840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572638464:192.168.203.129@tcp:38:kworker.0 00000800:00000100:2.0:1787829822.850824:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787829822.850836:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829822/real 1787829822] req@ffff8cb999c6b840 x1874672572638464/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829847 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829822.850844:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c6b840 x1874672572638464/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829847 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829822.850850:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829822.850852:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787829822.850854:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c6b840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572638464:192.168.203.129@tcp:38:kworker.0 00000100:00100000:2.0:1787829822.850857:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c68f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572638592:192.168.203.129@tcp:38:kworker.0 00000800:00000100:2.0:1787829822.850869:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787829822.850875:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829822/real 1787829822] req@ffff8cb999c68f00 x1874672572638592/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829847 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829822.850896:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c68f00 x1874672572638592/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829847 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829822.850905:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829822.850908:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787829822.850911:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c68f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572638592:192.168.203.129@tcp:38:kworker.0 00010000:00010000:2.0:1787829823.682508:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829824.706456:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829825.730370:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829826.754594:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829827.778449:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787829827.970383:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829827.970393:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787829827.970397:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829827.970406:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 2780 00000100:00080000:1.0:1787829827.970433:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829827.970440:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:1.0:1787829827.970480:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f38f00 x1874672572638720/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829827.970512:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829827.970516:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787829827.970518:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829827.970522:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 2780 00000100:00080000:1.0:1787829827.970533:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829827.970543:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:1.0:1787829827.970555:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787829827.970573:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f38b40 x1874672572638848/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829827.970586:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829827.970590:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1787829827.970592:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829827.970602:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 2780 00000100:00100000:2.0:1787829827.970603:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f38f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572638720:192.168.203.129@tcp:250:kworker.0 00000100:00080000:1.0:1787829827.970608:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829827.970622:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:1.0:1787829827.970629:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787829827.970646:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f3b0c0 x1874672572638976/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1787829827.970699:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787829827.970758:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829827/real 1787829827] req@ffff8cb999f38f00 x1874672572638720/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829852 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829827.970781:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f38f00 x1874672572638720/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829852 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829827.970792:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829827.970797:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787829827.970801:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f38f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572638720:192.168.203.129@tcp:250:kworker.0 00000100:00100000:2.0:1787829827.970832:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f38b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572638848:192.168.203.129@tcp:38:kworker.0 00000800:00000100:2.0:1787829827.970867:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787829827.970879:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829827/real 1787829827] req@ffff8cb999f38b40 x1874672572638848/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829852 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829827.970890:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f38b40 x1874672572638848/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829852 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829827.970898:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829827.970901:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787829827.970904:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f38b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572638848:192.168.203.129@tcp:38:kworker.0 00000100:00100000:2.0:1787829827.970922:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f3b0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572638976:192.168.203.129@tcp:38:kworker.0 00000800:00000100:2.0:1787829827.970943:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787829827.970952:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829827/real 1787829827] req@ffff8cb999f3b0c0 x1874672572638976/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829852 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829827.970963:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f3b0c0 x1874672572638976/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829852 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829827.970969:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829827.970972:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787829827.970975:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f3b0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572638976:192.168.203.129@tcp:38:kworker.0 00010000:00010000:2.0:1787829828.802635:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829829.826456:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829830.850461:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829831.874460:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829832.898389:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787829833.090393:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829833.090423:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787829833.090428:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829833.090438:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 2785 00000100:00080000:1.0:1787829833.090449:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829833.090456:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:1.0:1787829833.090490:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f3b480 x1874672572639104/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829833.090513:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829833.090516:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787829833.090518:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829833.090522:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 2785 00000100:00080000:1.0:1787829833.090525:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829833.090530:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:1.0:1787829833.090542:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787829833.090551:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f3b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572639104:192.168.203.129@tcp:250:kworker.0 00000100:00100000:1.0:1787829833.090559:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f39a40 x1874672572639232/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829833.090571:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829833.090575:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1787829833.090577:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829833.090581:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 2785 00000100:00080000:1.0:1787829833.090585:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829833.090589:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:1.0:1787829833.090595:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787829833.090607:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f39e00 x1874672572639360/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1787829833.090621:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787829833.090685:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829833/real 1787829833] req@ffff8cb999f3b480 x1874672572639104/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829858 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829833.090704:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f3b480 x1874672572639104/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829858 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829833.090712:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829833.090715:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787829833.090718:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f3b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572639104:192.168.203.129@tcp:250:kworker.0 00000100:00100000:2.0:1787829833.090745:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f39a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572639232:192.168.203.129@tcp:38:kworker.0 00000800:00000100:2.0:1787829833.090762:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787829833.090771:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829833/real 1787829833] req@ffff8cb999f39a40 x1874672572639232/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829858 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829833.090779:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f39a40 x1874672572639232/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829858 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829833.090784:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829833.090786:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787829833.090788:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f39a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572639232:192.168.203.129@tcp:38:kworker.0 00000100:00100000:2.0:1787829833.090791:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f39e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572639360:192.168.203.129@tcp:38:kworker.0 00000800:00000100:2.0:1787829833.090805:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787829833.090824:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829833/real 1787829833] req@ffff8cb999f39e00 x1874672572639360/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829858 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829833.090837:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f39e00 x1874672572639360/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829858 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829833.090845:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829833.090848:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787829833.090851:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f39e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572639360:192.168.203.129@tcp:38:kworker.0 00010000:00010000:2.0:1787829833.922430:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829834.946438:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829835.970440:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829836.994447:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829838.018436:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787829838.210406:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829838.210417:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787829838.210421:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829838.210435:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 2790 00000100:00080000:1.0:1787829838.210447:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829838.210455:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:1.0:1787829838.210499:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f3b840 x1874672572639488/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829838.210524:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829838.210528:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787829838.210529:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829838.210531:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 2790 00000100:00080000:1.0:1787829838.210534:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829838.210536:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:1.0:1787829838.210545:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787829838.210556:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f39e00 x1874672572639616/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829838.210564:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829838.210566:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1787829838.210567:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829838.210572:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 2790 00000100:00080000:1.0:1787829838.210574:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1787829838.210576:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f3b840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572639488:192.168.203.129@tcp:250:kworker.0 00000100:00080000:1.0:1787829838.210577:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:1.0:1787829838.210597:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787829838.210612:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f39a40 x1874672572639744/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1787829838.210652:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787829838.210697:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829838/real 1787829838] req@ffff8cb999f3b840 x1874672572639488/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829863 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829838.210712:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f3b840 x1874672572639488/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829863 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829838.210720:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829838.210723:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787829838.210726:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f3b840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572639488:192.168.203.129@tcp:250:kworker.0 00000100:00100000:2.0:1787829838.210747:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f39e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572639616:192.168.203.129@tcp:38:kworker.0 00000800:00000100:2.0:1787829838.210763:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787829838.210774:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829838/real 1787829838] req@ffff8cb999f39e00 x1874672572639616/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829863 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829838.210797:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f39e00 x1874672572639616/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829863 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829838.210805:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829838.210809:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787829838.210812:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f39e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572639616:192.168.203.129@tcp:38:kworker.0 00000100:00100000:2.0:1787829838.210819:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f39a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572639744:192.168.203.129@tcp:38:kworker.0 00000800:00000100:2.0:1787829838.210834:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787829838.210841:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829838/real 1787829838] req@ffff8cb999f39a40 x1874672572639744/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829863 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829838.210856:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f39a40 x1874672572639744/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829863 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829838.210861:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829838.210863:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787829838.210865:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f39a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572639744:192.168.203.129@tcp:38:kworker.0 00010000:00010000:2.0:1787829839.042429:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829840.066452:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829841.090461:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829842.114394:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829843.138410:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787829843.330433:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829843.330442:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787829843.330446:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829843.330455:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 2795 00000100:00080000:0.0:1787829843.330479:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829843.330486:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:0.0:1787829843.330515:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c6bc00 x1874672572639872/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829843.330543:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829843.330547:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787829843.330549:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829843.330554:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 2795 00000100:00080000:0.0:1787829843.330558:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829843.330562:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:0.0:1787829843.330573:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787829843.330588:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c69e00 x1874672572640000/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829843.330603:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1787829843.330605:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c6bc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572639872:192.168.203.129@tcp:250:kworker.0 00000100:00080000:0.0:1787829843.330606:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1787829843.330612:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829843.330622:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 2795 00000100:00080000:0.0:1787829843.330626:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829843.330629:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:0.0:1787829843.330634:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787829843.330650:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c6b480 x1874672572640128/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787829843.330731:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c69e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572640000:192.168.203.129@tcp:38:kworker.0 00000100:00100000:2.0:1787829843.330771:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c6b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572640128:192.168.203.129@tcp:38:kworker.0 00010000:00010000:2.0:1787829844.162446:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829845.186436:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829846.210448:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:2.0:1787829846.402665:0:2382:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.129:988 00000400:00000100:2.0:1787829846.402738:0:2382:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.129@tcp at host 192.168.203.129 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:2.0:1787829846.402761:0:2382:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.129@tcp 00000800:00000100:2.0:1787829846.402808:0:2382:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.129@tcp 00000800:00000100:2.0:1787829846.402821:0:2382:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.129@tcp 00000100:00100000:0.0:1787829846.402857:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829843/real 1787829846] req@ffff8cb999c69e00 x1874672572640000/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829868 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829846.402901:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c69e00 x1874672572640000/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829868 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829846.402916:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829846.402924:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787829846.402930:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c69e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572640000:192.168.203.129@tcp:38:kworker.0 00000100:00100000:0.0:1787829846.402939:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829843/real 1787829846] req@ffff8cb999c6b480 x1874672572640128/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829868 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829846.402955:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c6b480 x1874672572640128/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829868 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829846.402961:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829846.402964:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787829846.402966:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c6b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572640128:192.168.203.129@tcp:38:kworker.0 00000100:00100000:0.0:1787829846.402972:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829843/real 1787829846] req@ffff8cb999c6bc00 x1874672572639872/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829868 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829846.402984:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c6bc00 x1874672572639872/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829868 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829846.402993:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829846.402996:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787829846.402998:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c6bc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572639872:192.168.203.129@tcp:250:kworker.0 00010000:00010000:2.0:1787829847.234459:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829848.258461:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787829848.450372:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829848.450403:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787829848.450409:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829848.450422:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 2800 00000100:00080000:0.0:1787829848.450436:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829848.450443:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:0.0:1787829848.450490:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c6bc00 x1874672572640256/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829848.450521:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829848.450527:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787829848.450529:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829848.450534:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 2800 00000100:00080000:0.0:1787829848.450538:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829848.450542:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:0.0:1787829848.450555:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787829848.450569:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c6b480 x1874672572640384/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787829848.450576:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c6bc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572640256:192.168.203.129@tcp:250:kworker.0 00000100:00080000:0.0:1787829848.450583:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829848.450588:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1787829848.450590:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829848.450595:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 2800 00000100:00080000:0.0:1787829848.450598:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829848.450602:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:0.0:1787829848.450607:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787829848.450620:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c69e00 x1874672572640512/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1787829848.450663:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787829848.450743:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829848/real 1787829848] req@ffff8cb999c6bc00 x1874672572640256/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829873 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829848.450764:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c6bc00 x1874672572640256/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829873 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829848.450777:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829848.450783:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787829848.450789:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c6bc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572640256:192.168.203.129@tcp:250:kworker.0 00000100:00100000:2.0:1787829848.450824:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c6b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572640384:192.168.203.129@tcp:38:kworker.0 00000800:00000100:2.0:1787829848.450849:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787829848.450863:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829848/real 1787829848] req@ffff8cb999c6b480 x1874672572640384/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829873 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829848.450876:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c6b480 x1874672572640384/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829873 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829848.450884:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829848.450888:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787829848.450891:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c6b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572640384:192.168.203.129@tcp:38:kworker.0 00000100:00100000:2.0:1787829848.450897:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c69e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572640512:192.168.203.129@tcp:38:kworker.0 00000800:00000100:2.0:1787829848.450915:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787829848.450926:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829848/real 1787829848] req@ffff8cb999c69e00 x1874672572640512/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829873 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829848.450952:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c69e00 x1874672572640512/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829873 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829848.450960:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829848.450964:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787829848.450967:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c69e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572640512:192.168.203.129@tcp:38:kworker.0 00010000:00010000:2.0:1787829849.282460:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829850.306456:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829851.331026:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829852.354510:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829853.378539:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787829853.570363:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829853.570370:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787829853.570374:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829853.570382:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 2805 00000100:00080000:1.0:1787829853.570392:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829853.570400:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:1.0:1787829853.570433:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f3b480 x1874672572640640/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829853.570463:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829853.570466:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787829853.570468:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829853.570471:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 2805 00000100:00080000:1.0:1787829853.570474:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829853.570477:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:1.0:1787829853.570486:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787829853.570496:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f3b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572640640:192.168.203.129@tcp:250:kworker.0 00000100:00100000:1.0:1787829853.570497:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f3b0c0 x1874672572640768/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829853.570511:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829853.570514:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1787829853.570516:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829853.570519:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 2805 00000100:00080000:1.0:1787829853.570536:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829853.570540:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:1.0:1787829853.570545:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787829853.570555:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f38b40 x1874672572640896/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1787829853.570576:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787829853.570631:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829853/real 1787829853] req@ffff8cb999f3b480 x1874672572640640/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829878 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829853.570646:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f3b480 x1874672572640640/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829878 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829853.570653:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829853.570656:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787829853.570660:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f3b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572640640:192.168.203.129@tcp:250:kworker.0 00000100:00100000:0.0:1787829853.570696:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f3b0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572640768:192.168.203.129@tcp:38:kworker.0 00000800:00000100:0.0:1787829853.570715:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787829853.570724:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829853/real 1787829853] req@ffff8cb999f3b0c0 x1874672572640768/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829878 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829853.570734:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f3b0c0 x1874672572640768/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829878 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829853.570740:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829853.570743:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787829853.570745:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f3b0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572640768:192.168.203.129@tcp:38:kworker.0 00000100:00100000:0.0:1787829853.570749:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f38b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572640896:192.168.203.129@tcp:38:kworker.0 00000800:00000100:0.0:1787829853.570768:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787829853.570785:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829853/real 1787829853] req@ffff8cb999f38b40 x1874672572640896/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829878 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829853.570795:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f38b40 x1874672572640896/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829878 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829853.570800:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829853.570803:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787829853.570805:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f38b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572640896:192.168.203.129@tcp:38:kworker.0 00010000:00010000:2.0:1787829854.402772:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829855.426857:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829856.450539:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829857.474473:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829858.498455:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787829858.690438:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829858.690452:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787829858.690458:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829858.690471:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 2810 00000100:00080000:1.0:1787829858.690486:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829858.690495:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:1.0:1787829858.690549:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f38f00 x1874672572641024/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829858.690580:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829858.690584:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787829858.690586:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829858.690591:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 2810 00000100:00080000:1.0:1787829858.690596:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829858.690600:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:1.0:1787829858.690612:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787829858.690616:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f38f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572641024:192.168.203.129@tcp:250:kworker.0 00000100:00100000:1.0:1787829858.690630:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f392c0 x1874672572641152/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829858.690659:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829858.690681:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1787829858.690683:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829858.690688:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 2810 00000100:00080000:1.0:1787829858.690695:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829858.690700:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:1.0:1787829858.690707:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000800:00000100:0.0:1787829858.690710:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787829858.690724:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f3a580 x1874672572641280/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787829858.690767:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829858/real 1787829858] req@ffff8cb999f38f00 x1874672572641024/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829883 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829858.690788:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f38f00 x1874672572641024/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829883 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829858.690801:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829858.690806:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787829858.690811:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f38f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572641024:192.168.203.129@tcp:250:kworker.0 00000100:00100000:0.0:1787829858.690842:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f392c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572641152:192.168.203.129@tcp:38:kworker.0 00000800:00000100:0.0:1787829858.690863:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787829858.690873:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829858/real 1787829858] req@ffff8cb999f392c0 x1874672572641152/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829883 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829858.690882:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f392c0 x1874672572641152/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829883 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829858.690899:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829858.690902:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787829858.690904:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f392c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572641152:192.168.203.129@tcp:38:kworker.0 00000100:00100000:0.0:1787829858.690909:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f3a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572641280:192.168.203.129@tcp:38:kworker.0 00000800:00000100:0.0:1787829858.690924:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787829858.690931:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829858/real 1787829858] req@ffff8cb999f3a580 x1874672572641280/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829883 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829858.690939:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f3a580 x1874672572641280/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829883 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829858.690945:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829858.690949:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787829858.690953:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f3a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572641280:192.168.203.129@tcp:38:kworker.0 00010000:00010000:2.0:1787829859.522419:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829860.546457:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829861.570425:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829862.594459:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829863.618463:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787829863.810389:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829863.810400:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787829863.810405:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829863.810415:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 2815 00000100:00080000:0.0:1787829863.810428:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829863.810434:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:0.0:1787829863.810478:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c692c0 x1874672572641408/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829863.810505:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829863.810529:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787829863.810531:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829863.810536:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 2815 00000100:00080000:0.0:1787829863.810540:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829863.810545:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:2.0:1787829863.810551:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c692c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572641408:192.168.203.129@tcp:250:kworker.0 00000008:00100000:0.0:1787829863.810556:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787829863.810569:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c69680 x1874672572641536/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829863.810580:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829863.810583:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1787829863.810584:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829863.810588:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 2815 00000100:00080000:0.0:1787829863.810591:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829863.810595:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:0.0:1787829863.810611:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787829863.810623:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c6a1c0 x1874672572641664/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1787829863.810640:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787829863.810695:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829863/real 1787829863] req@ffff8cb999c692c0 x1874672572641408/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829888 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829863.810715:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c692c0 x1874672572641408/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829888 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829863.810735:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829863.810740:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787829863.810745:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c692c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572641408:192.168.203.129@tcp:250:kworker.0 00000100:00100000:2.0:1787829863.810773:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c69680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572641536:192.168.203.129@tcp:38:kworker.0 00000800:00000100:2.0:1787829863.810813:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787829863.810829:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829863/real 1787829863] req@ffff8cb999c69680 x1874672572641536/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829888 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829863.810841:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c69680 x1874672572641536/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829888 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829863.810848:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829863.810850:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787829863.810852:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c69680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572641536:192.168.203.129@tcp:38:kworker.0 00000100:00100000:2.0:1787829863.810857:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c6a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572641664:192.168.203.129@tcp:38:kworker.0 00000800:00000100:2.0:1787829863.810873:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787829863.810881:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829863/real 1787829863] req@ffff8cb999c6a1c0 x1874672572641664/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829888 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829863.810888:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c6a1c0 x1874672572641664/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829888 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829863.810894:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829863.810896:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787829863.810898:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c6a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572641664:192.168.203.129@tcp:38:kworker.0 00010000:00010000:2.0:1787829864.642442:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829865.666488:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829866.690476:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829867.714399:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829868.738486:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787829868.930387:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829868.930399:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787829868.930404:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829868.930414:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 2821 00000100:00080000:2.0:1787829868.930425:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829868.930432:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:2.0:1787829868.930474:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8cb40 x1874672572641792/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829868.930499:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829868.930502:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787829868.930504:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829868.930506:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 2821 00000100:00080000:2.0:1787829868.930509:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829868.930511:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:2.0:1787829868.930521:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787829868.930530:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8c000 x1874672572641920/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829868.930539:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829868.930543:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00100000:0.0:1787829868.930550:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8cb40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572641792:192.168.203.129@tcp:250:kworker.0 00000100:00080000:2.0:1787829868.930575:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829868.930582:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 2821 00000100:00080000:2.0:1787829868.930586:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829868.930591:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:2.0:1787829868.930597:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787829868.930612:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8f480 x1874672572642048/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1787829868.930662:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787829868.930723:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829868/real 1787829868] req@ffff8cb998a8cb40 x1874672572641792/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829893 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829868.930744:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8cb40 x1874672572641792/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829893 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829868.930770:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829868.930775:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787829868.930780:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8cb40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572641792:192.168.203.129@tcp:250:kworker.0 00000100:00100000:0.0:1787829868.930811:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572641920:192.168.203.129@tcp:38:kworker.0 00000800:00000100:0.0:1787829868.930839:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787829868.930852:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829868/real 1787829868] req@ffff8cb998a8c000 x1874672572641920/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829893 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829868.930866:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8c000 x1874672572641920/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829893 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829868.930874:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829868.930877:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787829868.930880:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572641920:192.168.203.129@tcp:38:kworker.0 00000100:00100000:0.0:1787829868.930886:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572642048:192.168.203.129@tcp:38:kworker.0 00000800:00000100:0.0:1787829868.930906:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787829868.930917:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829868/real 1787829868] req@ffff8cb998a8f480 x1874672572642048/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829893 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829868.930930:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8f480 x1874672572642048/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829893 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829868.930940:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829868.930943:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787829868.930947:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572642048:192.168.203.129@tcp:38:kworker.0 00010000:00010000:0.0:1787829869.762446:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829870.786463:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829871.810434:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787829872.834474:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829873.858496:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787829874.050439:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829874.050455:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787829874.050461:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829874.050474:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 2826 00000100:00080000:1.0:1787829874.050488:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829874.050498:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:1.0:1787829874.050565:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f383c0 x1874672572642176/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829874.050600:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829874.050605:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787829874.050607:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829874.050611:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 2826 00000100:00080000:1.0:1787829874.050615:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829874.050621:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:1.0:1787829874.050634:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787829874.050654:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f383c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572642176:192.168.203.129@tcp:250:kworker.0 00000100:00100000:1.0:1787829874.050655:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f3a580 x1874672572642304/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829874.050689:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829874.050693:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1787829874.050696:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829874.050701:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 2826 00000100:00080000:1.0:1787829874.050718:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829874.050724:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000800:00000100:0.0:1787829874.050730:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000008:00100000:1.0:1787829874.050734:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787829874.050754:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f392c0 x1874672572642432/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787829874.050776:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829874/real 1787829874] req@ffff8cb999f383c0 x1874672572642176/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829899 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829874.050794:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f383c0 x1874672572642176/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829899 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829874.050804:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829874.050807:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787829874.050810:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f383c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572642176:192.168.203.129@tcp:250:kworker.0 00000100:00100000:0.0:1787829874.050835:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f3a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572642304:192.168.203.129@tcp:38:kworker.0 00000800:00000100:0.0:1787829874.050850:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787829874.050859:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829874/real 1787829874] req@ffff8cb999f3a580 x1874672572642304/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829899 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829874.050867:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f3a580 x1874672572642304/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829899 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829874.050873:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829874.050875:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787829874.050877:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f3a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572642304:192.168.203.129@tcp:38:kworker.0 00000100:00100000:0.0:1787829874.050881:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f392c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572642432:192.168.203.129@tcp:38:kworker.0 00000800:00000100:0.0:1787829874.050903:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787829874.050910:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829874/real 1787829874] req@ffff8cb999f392c0 x1874672572642432/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829899 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829874.050917:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f392c0 x1874672572642432/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829899 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829874.050922:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829874.050924:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787829874.050926:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f392c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572642432:192.168.203.129@tcp:38:kworker.0 00010000:00010000:2.0:1787829874.882480:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829875.906465:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829876.930468:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829877.954454:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829878.978410:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787829879.170375:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829879.170386:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787829879.170390:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829879.170401:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 2831 00000100:00080000:1.0:1787829879.170414:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829879.170422:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:1.0:1787829879.170464:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f38f00 x1874672572642560/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829879.170485:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829879.170489:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787829879.170490:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829879.170509:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 2831 00000100:00080000:1.0:1787829879.170512:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829879.170515:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:0.0:1787829879.170529:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f38f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572642560:192.168.203.129@tcp:250:kworker.0 00000008:00100000:1.0:1787829879.170546:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787829879.170559:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f38b40 x1874672572642688/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829879.170568:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829879.170571:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1787829879.170572:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829879.170575:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 2831 00000100:00080000:1.0:1787829879.170578:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829879.170581:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:1.0:1787829879.170585:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787829879.170591:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f3b0c0 x1874672572642816/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1787829879.170611:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787829879.170661:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829879/real 1787829879] req@ffff8cb999f38f00 x1874672572642560/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829904 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829879.170677:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f38f00 x1874672572642560/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829904 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829879.170685:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829879.170688:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787829879.170691:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f38f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572642560:192.168.203.129@tcp:250:kworker.0 00000100:00100000:0.0:1787829879.170714:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f38b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572642688:192.168.203.129@tcp:38:kworker.0 00000800:00000100:0.0:1787829879.170729:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787829879.170740:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829879/real 1787829879] req@ffff8cb999f38b40 x1874672572642688/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829904 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829879.170770:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f38b40 x1874672572642688/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829904 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829879.170780:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829879.170784:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787829879.170788:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f38b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572642688:192.168.203.129@tcp:38:kworker.0 00000100:00100000:0.0:1787829879.170795:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f3b0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572642816:192.168.203.129@tcp:38:kworker.0 00000800:00000100:0.0:1787829879.170812:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787829879.170820:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829879/real 1787829879] req@ffff8cb999f3b0c0 x1874672572642816/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829904 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829879.170828:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f3b0c0 x1874672572642816/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829904 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829879.170834:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829879.170836:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787829879.170838:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f3b0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572642816:192.168.203.129@tcp:38:kworker.0 00010000:00010000:2.0:1787829880.002452:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829881.026466:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829882.050523:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829883.074488:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829884.098473:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787829884.290782:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829884.290792:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787829884.290798:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829884.290809:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 2836 00000100:00080000:0.0:1787829884.290821:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829884.290828:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:0.0:1787829884.290873:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c68000 x1874672572642944/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829884.290923:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829884.290927:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787829884.290928:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829884.290931:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 2836 00000100:00080000:0.0:1787829884.290934:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829884.290938:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:0.0:1787829884.290948:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787829884.290960:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c6a580 x1874672572643072/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829884.290973:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1787829884.290979:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c68000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572642944:192.168.203.129@tcp:250:kworker.0 00000100:00080000:0.0:1787829884.290980:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1787829884.290984:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829884.290991:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 2836 00000100:00080000:0.0:1787829884.290996:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829884.291001:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:0.0:1787829884.291007:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787829884.291022:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c6a1c0 x1874672572643200/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1787829884.291052:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787829884.291109:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829884/real 1787829884] req@ffff8cb999c68000 x1874672572642944/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829909 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829884.291126:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c68000 x1874672572642944/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829909 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829884.291135:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829884.291149:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787829884.291153:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c68000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572642944:192.168.203.129@tcp:250:kworker.0 00000100:00100000:2.0:1787829884.291178:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c6a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572643072:192.168.203.129@tcp:38:kworker.0 00000800:00000100:2.0:1787829884.291197:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787829884.291213:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829884/real 1787829884] req@ffff8cb999c6a580 x1874672572643072/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829909 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829884.291222:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c6a580 x1874672572643072/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829909 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829884.291228:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829884.291230:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787829884.291233:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c6a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572643072:192.168.203.129@tcp:38:kworker.0 00000100:00100000:2.0:1787829884.291237:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c6a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572643200:192.168.203.129@tcp:38:kworker.0 00000800:00000100:2.0:1787829884.291251:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787829884.291257:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829884/real 1787829884] req@ffff8cb999c6a1c0 x1874672572643200/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829909 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829884.291314:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c6a1c0 x1874672572643200/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829909 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829884.291321:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829884.291323:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787829884.291326:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c6a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572643200:192.168.203.129@tcp:38:kworker.0 00010000:00010000:2.0:1787829885.122415:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829886.146458:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829887.170458:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829888.194446:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829889.218413:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787829889.410437:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829889.410450:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787829889.410458:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829889.410474:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 2841 00000100:00080000:2.0:1787829889.410488:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829889.410494:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:2.0:1787829889.410539:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8d680 x1874672572643328/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829889.410567:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829889.410570:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787829889.410573:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829889.410576:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 2841 00000100:00080000:2.0:1787829889.410580:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829889.410585:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:2.0:1787829889.410597:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787829889.410610:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8e580 x1874672572643456/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829889.410619:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829889.410624:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1787829889.410626:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829889.410631:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 2841 00000100:00080000:2.0:1787829889.410655:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829889.410665:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:3.0:1787829889.410665:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8d680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572643328:192.168.203.129@tcp:250:kworker.0 00000008:00100000:2.0:1787829889.410678:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787829889.410691:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8cf00 x1874672572643584/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1787829889.410960:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:3.0:1787829889.411060:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829889/real 1787829889] req@ffff8cb998a8d680 x1874672572643328/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829914 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787829889.411086:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8d680 x1874672572643328/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829914 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787829889.411100:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829889.411105:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:3.0:1787829889.411109:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8d680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572643328:192.168.203.129@tcp:250:kworker.0 00000100:00100000:3.0:1787829889.411119:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8e580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572643456:192.168.203.129@tcp:38:kworker.0 00000800:00000100:3.0:1787829889.411145:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:3.0:1787829889.411159:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829889/real 1787829889] req@ffff8cb998a8e580 x1874672572643456/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829914 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787829889.411172:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8e580 x1874672572643456/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829914 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787829889.411209:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829889.411214:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:3.0:1787829889.411217:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8e580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572643456:192.168.203.129@tcp:38:kworker.0 00000100:00100000:3.0:1787829889.411247:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8cf00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572643584:192.168.203.129@tcp:38:kworker.0 00000800:00000100:3.0:1787829889.411309:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:3.0:1787829889.411328:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829889/real 1787829889] req@ffff8cb998a8cf00 x1874672572643584/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829914 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787829889.411341:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8cf00 x1874672572643584/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829914 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787829889.411350:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829889.411353:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:3.0:1787829889.411357:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8cf00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572643584:192.168.203.129@tcp:38:kworker.0 00010000:00010000:2.0:1787829890.242430:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829891.266457:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829892.290510:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829893.314475:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829894.338462:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787829894.530418:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829894.530429:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787829894.530434:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829894.530444:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 2846 00000100:00080000:0.0:1787829894.530480:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829894.530487:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:0.0:1787829894.530528:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c69680 x1874672572643712/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829894.530555:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829894.530558:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787829894.530559:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829894.530562:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 2846 00000100:00080000:0.0:1787829894.530564:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829894.530567:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:0.0:1787829894.530576:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787829894.530585:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c692c0 x1874672572643840/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829894.530593:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829894.530595:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1787829894.530597:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829894.530600:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 2846 00000100:00080000:0.0:1787829894.530603:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829894.530605:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:0.0:1787829894.530608:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787829894.530614:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c69680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572643712:192.168.203.129@tcp:250:kworker.0 00000100:00100000:0.0:1787829894.530617:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c69e00 x1874672572643968/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1787829894.530701:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:3.0:1787829894.530765:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829894/real 1787829894] req@ffff8cb999c69680 x1874672572643712/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829919 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787829894.530787:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c69680 x1874672572643712/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829919 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787829894.530800:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829894.530804:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:3.0:1787829894.530815:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c69680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572643712:192.168.203.129@tcp:250:kworker.0 00000100:00100000:3.0:1787829894.530824:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c692c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572643840:192.168.203.129@tcp:38:kworker.0 00000800:00000100:3.0:1787829894.530848:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:3.0:1787829894.530861:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829894/real 1787829894] req@ffff8cb999c692c0 x1874672572643840/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829919 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787829894.530873:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c692c0 x1874672572643840/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829919 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787829894.530882:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829894.530885:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:3.0:1787829894.530888:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c692c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572643840:192.168.203.129@tcp:38:kworker.0 00000100:00100000:3.0:1787829894.530914:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c69e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572643968:192.168.203.129@tcp:38:kworker.0 00000800:00000100:3.0:1787829894.530932:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:3.0:1787829894.530943:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829894/real 1787829894] req@ffff8cb999c69e00 x1874672572643968/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829919 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787829894.530955:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c69e00 x1874672572643968/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829919 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787829894.530963:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829894.530966:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:3.0:1787829894.530970:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c69e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572643968:192.168.203.129@tcp:38:kworker.0 00010000:00010000:1.0:1787829895.362547:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829896.386464:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829897.410561:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787829898.434410:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829899.458417:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787829899.650328:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829899.650349:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787829899.650364:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829899.650372:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 2851 00000100:00080000:0.0:1787829899.650382:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829899.650390:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:0.0:1787829899.650421:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c6b480 x1874672572644096/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829899.650444:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829899.650448:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787829899.650449:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829899.650452:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 2851 00000100:00080000:0.0:1787829899.650454:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829899.650457:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:0.0:1787829899.650465:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787829899.650475:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c6bc00 x1874672572644224/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1787829899.650481:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c6b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572644096:192.168.203.129@tcp:250:kworker.0 00000100:00080000:0.0:1787829899.650492:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829899.650494:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1787829899.650495:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829899.650498:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 2851 00000100:00080000:0.0:1787829899.650500:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829899.650502:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:0.0:1787829899.650504:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787829899.650511:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c68f00 x1874672572644352/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1787829899.650555:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:3.0:1787829899.650589:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829899/real 1787829899] req@ffff8cb999c6b480 x1874672572644096/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829924 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787829899.650601:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c6b480 x1874672572644096/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829924 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787829899.650608:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829899.650610:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:3.0:1787829899.650613:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c6b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572644096:192.168.203.129@tcp:250:kworker.0 00000100:00100000:3.0:1787829899.650631:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c6bc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572644224:192.168.203.129@tcp:38:kworker.0 00000800:00000100:3.0:1787829899.650645:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:3.0:1787829899.650653:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829899/real 1787829899] req@ffff8cb999c6bc00 x1874672572644224/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829924 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787829899.650662:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c6bc00 x1874672572644224/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829924 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787829899.650670:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829899.650673:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:3.0:1787829899.650676:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c6bc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572644224:192.168.203.129@tcp:38:kworker.0 00000100:00100000:3.0:1787829899.650682:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c68f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572644352:192.168.203.129@tcp:38:kworker.0 00000800:00000100:3.0:1787829899.650708:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:3.0:1787829899.650724:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829899/real 1787829899] req@ffff8cb999c68f00 x1874672572644352/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829924 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787829899.650731:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c68f00 x1874672572644352/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829924 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787829899.650745:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829899.650747:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:3.0:1787829899.650749:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c68f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572644352:192.168.203.129@tcp:38:kworker.0 00010000:00010000:2.0:1787829900.482439:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787829901.506433:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829902.530455:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829903.554471:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787829904.578427:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787829904.770433:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829904.770446:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787829904.770451:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829904.770464:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 2856 00000100:00080000:3.0:1787829904.770489:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829904.770500:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:3.0:1787829904.770575:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb9923161c0 x1874672572644480/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787829904.770606:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829904.770610:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787829904.770612:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829904.770616:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 2856 00000100:00080000:3.0:1787829904.770621:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829904.770626:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:3.0:1787829904.770638:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787829904.770644:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb9923161c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572644480:192.168.203.129@tcp:250:kworker.0 00000100:00100000:3.0:1787829904.770657:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb9923170c0 x1874672572644608/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787829904.770670:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829904.770674:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1787829904.770694:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829904.770699:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 2856 00000100:00080000:3.0:1787829904.770703:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829904.770708:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:3.0:1787829904.770713:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000800:00000100:0.0:1787829904.770721:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:3.0:1787829904.770728:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb992315a40 x1874672572644736/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787829904.770772:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829904/real 1787829904] req@ffff8cb9923161c0 x1874672572644480/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829929 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829904.770786:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb9923161c0 x1874672572644480/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829929 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829904.770795:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829904.770798:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787829904.770801:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb9923161c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572644480:192.168.203.129@tcp:250:kworker.0 00000100:00100000:0.0:1787829904.770823:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb9923170c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572644608:192.168.203.129@tcp:38:kworker.0 00000800:00000100:0.0:1787829904.770837:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787829904.770846:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829904/real 1787829904] req@ffff8cb9923170c0 x1874672572644608/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829929 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829904.770853:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb9923170c0 x1874672572644608/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829929 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829904.770858:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829904.770860:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787829904.770862:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb9923170c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572644608:192.168.203.129@tcp:38:kworker.0 00000100:00100000:0.0:1787829904.770865:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb992315a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572644736:192.168.203.129@tcp:38:kworker.0 00000800:00000100:0.0:1787829904.770883:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787829904.770890:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829904/real 1787829904] req@ffff8cb992315a40 x1874672572644736/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829929 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829904.770896:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb992315a40 x1874672572644736/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829929 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829904.770901:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829904.770902:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787829904.770904:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb992315a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572644736:192.168.203.129@tcp:38:kworker.0 00010000:00010000:2.0:1787829905.602441:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829906.626463:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829907.650445:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829908.674495:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829909.698478:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787829909.890381:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829909.890394:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787829909.890399:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829909.890411:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 2862 00000100:00080000:3.0:1787829909.890424:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829909.890432:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:3.0:1787829909.890475:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb992314f00 x1874672572644864/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787829909.890508:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829909.890512:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787829909.890514:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829909.890518:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 2862 00000100:00080000:3.0:1787829909.890522:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829909.890526:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:3.0:1787829909.890545:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787829909.890553:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb992314f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572644864:192.168.203.129@tcp:250:kworker.0 00000100:00100000:3.0:1787829909.890562:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb992316940 x1874672572644992/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787829909.890598:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829909.890602:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1787829909.890604:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829909.890608:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 2862 00000100:00080000:3.0:1787829909.890613:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829909.890617:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:3.0:1787829909.890624:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787829909.890636:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb992315e00 x1874672572645120/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787829909.890677:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb992316940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572644992:192.168.203.129@tcp:38:kworker.0 00000100:00100000:0.0:1787829909.890705:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb992315e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572645120:192.168.203.129@tcp:38:kworker.0 00010000:00010000:2.0:1787829910.722418:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829911.746489:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829912.770518:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:3.0:1787829912.962654:0:2383:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.129:988 00000400:00000100:3.0:1787829912.962713:0:2383:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.129@tcp at host 192.168.203.129 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:3.0:1787829912.962738:0:2383:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.129@tcp 00000800:00000100:3.0:1787829912.962786:0:2383:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.129@tcp 00000800:00000100:3.0:1787829912.962798:0:2383:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.129@tcp 00000100:00100000:0.0:1787829912.963376:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829909/real 1787829912] req@ffff8cb992316940 x1874672572644992/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829934 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829912.963406:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb992316940 x1874672572644992/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829934 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829912.963418:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829912.963425:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787829912.963429:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb992316940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572644992:192.168.203.129@tcp:38:kworker.0 00000100:00100000:0.0:1787829912.963437:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829909/real 1787829912] req@ffff8cb992315e00 x1874672572645120/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829934 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829912.963481:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb992315e00 x1874672572645120/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829934 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829912.963489:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829912.963494:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787829912.963496:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb992315e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572645120:192.168.203.129@tcp:38:kworker.0 00000100:00100000:0.0:1787829912.963503:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829909/real 1787829912] req@ffff8cb992314f00 x1874672572644864/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829934 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829912.963513:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb992314f00 x1874672572644864/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829934 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829912.963520:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829912.963522:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787829912.963537:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb992314f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572644864:192.168.203.129@tcp:250:kworker.0 00010000:00010000:2.0:1787829913.794467:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829914.818459:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787829915.010402:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829915.010416:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787829915.010420:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829915.010433:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 2867 00000100:00080000:3.0:1787829915.010446:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829915.010454:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:3.0:1787829915.010494:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb992314780 x1874672572645248/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787829915.010529:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829915.010549:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787829915.010552:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829915.010556:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 2867 00000100:00080000:3.0:1787829915.010561:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829915.010567:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:3.0:1787829915.010580:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787829915.010587:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb992314780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572645248:192.168.203.129@tcp:250:kworker.0 00000100:00100000:3.0:1787829915.010594:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb992314b40 x1874672572645376/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787829915.010609:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829915.010613:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1787829915.010615:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829915.010618:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 2867 00000100:00080000:3.0:1787829915.010622:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829915.010625:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:3.0:1787829915.010630:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787829915.010656:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb992317480 x1874672572645504/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1787829915.010681:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787829915.010735:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829915/real 1787829915] req@ffff8cb992314780 x1874672572645248/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829940 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829915.010765:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb992314780 x1874672572645248/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829940 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829915.010776:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829915.010780:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787829915.010784:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb992314780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572645248:192.168.203.129@tcp:250:kworker.0 00000100:00100000:0.0:1787829915.010821:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb992314b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572645376:192.168.203.129@tcp:38:kworker.0 00000800:00000100:0.0:1787829915.010848:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787829915.010861:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829915/real 1787829915] req@ffff8cb992314b40 x1874672572645376/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829940 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829915.010874:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb992314b40 x1874672572645376/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829940 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829915.010882:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829915.010885:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787829915.010888:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb992314b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572645376:192.168.203.129@tcp:38:kworker.0 00000100:00100000:0.0:1787829915.010894:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb992317480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572645504:192.168.203.129@tcp:38:kworker.0 00000800:00000100:0.0:1787829915.010915:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787829915.010927:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829915/real 1787829915] req@ffff8cb992317480 x1874672572645504/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829940 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829915.010937:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb992317480 x1874672572645504/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829940 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829915.010944:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829915.010947:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787829915.010950:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb992317480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572645504:192.168.203.129@tcp:38:kworker.0 00010000:00010000:2.0:1787829915.842426:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829916.866450:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829917.891286:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829918.914766:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829919.938526:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787829920.130367:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829920.130379:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787829920.130384:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829920.130396:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 2872 00000100:00080000:3.0:1787829920.130409:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829920.130417:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:3.0:1787829920.130460:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb992316580 x1874672572645632/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787829920.130510:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829920.130514:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787829920.130516:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829920.130520:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 2872 00000100:00080000:3.0:1787829920.130524:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829920.130528:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:3.0:1787829920.130540:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787829920.130557:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb992316580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572645632:192.168.203.129@tcp:250:kworker.0 00000100:00100000:3.0:1787829920.130558:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb992317480 x1874672572645760/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787829920.130573:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829920.130577:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1787829920.130578:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829920.130582:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 2872 00000100:00080000:3.0:1787829920.130585:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829920.130589:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:3.0:1787829920.130593:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787829920.130604:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb992314b40 x1874672572645888/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1787829920.130640:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787829920.130678:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829920/real 1787829920] req@ffff8cb992316580 x1874672572645632/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829945 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829920.130697:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb992316580 x1874672572645632/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829945 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829920.130705:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829920.130707:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787829920.130710:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb992316580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572645632:192.168.203.129@tcp:250:kworker.0 00000100:00100000:0.0:1787829920.130731:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb992317480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572645760:192.168.203.129@tcp:38:kworker.0 00000800:00000100:0.0:1787829920.130749:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787829920.130761:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829920/real 1787829920] req@ffff8cb992317480 x1874672572645760/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829945 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829920.130772:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb992317480 x1874672572645760/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829945 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829920.130779:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829920.130781:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787829920.130784:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb992317480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572645760:192.168.203.129@tcp:38:kworker.0 00000100:00100000:0.0:1787829920.130789:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb992314b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572645888:192.168.203.129@tcp:38:kworker.0 00000800:00000100:0.0:1787829920.130807:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787829920.130817:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829920/real 1787829920] req@ffff8cb992314b40 x1874672572645888/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829945 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829920.130828:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb992314b40 x1874672572645888/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829945 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829920.130844:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829920.130847:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787829920.130849:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb992314b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572645888:192.168.203.129@tcp:38:kworker.0 00010000:00010000:2.0:1787829920.963317:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829921.986669:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829923.010451:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829924.034472:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829925.058466:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787829925.250356:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829925.250366:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787829925.250371:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829925.250383:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 2877 00000100:00080000:3.0:1787829925.250413:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829925.250421:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:3.0:1787829925.250457:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb992314780 x1874672572646016/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787829925.250489:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829925.250493:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787829925.250496:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829925.250500:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 2877 00000100:00080000:3.0:1787829925.250505:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829925.250510:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:3.0:1787829925.250521:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787829925.250526:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb992314780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572646016:192.168.203.129@tcp:250:kworker.0 00000100:00100000:3.0:1787829925.250536:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb992314f00 x1874672572646144/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787829925.250547:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829925.250550:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1787829925.250552:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829925.250555:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 2877 00000100:00080000:3.0:1787829925.250558:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829925.250561:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:3.0:1787829925.250564:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787829925.250572:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb992315e00 x1874672572646272/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1787829925.250598:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787829925.250642:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829925/real 1787829925] req@ffff8cb992314780 x1874672572646016/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829950 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829925.250661:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb992314780 x1874672572646016/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829950 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829925.250673:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829925.250677:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787829925.250681:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb992314780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572646016:192.168.203.129@tcp:250:kworker.0 00000100:00100000:2.0:1787829925.250703:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb992314f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572646144:192.168.203.129@tcp:38:kworker.0 00000800:00000100:2.0:1787829925.250728:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787829925.250737:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829925/real 1787829925] req@ffff8cb992314f00 x1874672572646144/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829950 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829925.250747:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb992314f00 x1874672572646144/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829950 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829925.250756:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829925.250759:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787829925.250763:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb992314f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572646144:192.168.203.129@tcp:38:kworker.0 00000100:00100000:2.0:1787829925.250776:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb992315e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572646272:192.168.203.129@tcp:38:kworker.0 00000800:00000100:2.0:1787829925.250797:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787829925.250805:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829925/real 1787829925] req@ffff8cb992315e00 x1874672572646272/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829950 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829925.250813:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb992315e00 x1874672572646272/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829950 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829925.250818:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829925.250821:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787829925.250824:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb992315e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572646272:192.168.203.129@tcp:38:kworker.0 00010000:00010000:2.0:1787829926.082422:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829927.106450:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829928.130407:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829929.154447:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829930.178409:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787829930.370392:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829930.370429:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787829930.370433:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829930.370445:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 2882 00000100:00080000:3.0:1787829930.370460:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829930.370467:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:3.0:1787829930.370509:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb992316940 x1874672572646400/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787829930.370545:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829930.370548:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787829930.370550:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829930.370554:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 2882 00000100:00080000:3.0:1787829930.370558:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829930.370562:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:3.0:1787829930.370574:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787829930.370587:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb992315a40 x1874672572646528/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787829930.370596:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829930.370600:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1787829930.370602:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829930.370606:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 2882 00000100:00080000:3.0:1787829930.370609:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829930.370614:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:3.0:1787829930.370620:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787829930.370633:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb992316940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572646400:192.168.203.129@tcp:250:kworker.0 00000100:00100000:3.0:1787829930.370634:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb9923170c0 x1874672572646656/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1787829930.370712:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787829930.370754:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829930/real 1787829930] req@ffff8cb992316940 x1874672572646400/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829955 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829930.370769:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb992316940 x1874672572646400/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829955 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829930.370781:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829930.370785:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787829930.370789:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb992316940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572646400:192.168.203.129@tcp:250:kworker.0 00000100:00100000:2.0:1787829930.370809:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb992315a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572646528:192.168.203.129@tcp:38:kworker.0 00000800:00000100:2.0:1787829930.370831:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787829930.370845:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829930/real 1787829930] req@ffff8cb992315a40 x1874672572646528/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829955 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829930.370855:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb992315a40 x1874672572646528/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829955 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829930.370862:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829930.370865:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787829930.370868:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb992315a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572646528:192.168.203.129@tcp:38:kworker.0 00000100:00100000:2.0:1787829930.370893:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb9923170c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572646656:192.168.203.129@tcp:38:kworker.0 00000800:00000100:2.0:1787829930.370908:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787829930.370918:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829930/real 1787829930] req@ffff8cb9923170c0 x1874672572646656/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829955 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829930.370927:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb9923170c0 x1874672572646656/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829955 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829930.370934:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829930.370937:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787829930.370940:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb9923170c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572646656:192.168.203.129@tcp:38:kworker.0 00010000:00010000:0.0:1787829931.202475:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829932.226454:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829933.250576:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787829934.274463:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787829935.298428:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787829935.490365:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829935.490386:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787829935.490389:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829935.490397:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 2887 00000100:00080000:2.0:1787829935.490407:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829935.490415:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:2.0:1787829935.490445:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8c780 x1874672572646784/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829935.490468:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829935.490470:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787829935.490472:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829935.490474:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 2887 00000100:00080000:2.0:1787829935.490477:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829935.490479:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:2.0:1787829935.490487:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787829935.490496:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8f0c0 x1874672572646912/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829935.490503:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829935.490505:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1787829935.490506:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829935.490508:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 2887 00000100:00080000:2.0:1787829935.490510:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829935.490512:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:2.0:1787829935.490515:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787829935.490523:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8fc00 x1874672572647040/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787829935.490525:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8c780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572646784:192.168.203.129@tcp:250:kworker.0 00000800:00000100:0.0:1787829935.490589:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787829935.490629:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829935/real 1787829935] req@ffff8cb998a8c780 x1874672572646784/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829960 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829935.490646:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8c780 x1874672572646784/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829960 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829935.490656:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829935.490660:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787829935.490664:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8c780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572646784:192.168.203.129@tcp:250:kworker.0 00000100:00100000:0.0:1787829935.490671:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8f0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572646912:192.168.203.129@tcp:38:kworker.0 00000800:00000100:0.0:1787829935.490692:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787829935.490704:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829935/real 1787829935] req@ffff8cb998a8f0c0 x1874672572646912/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829960 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829935.490715:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8f0c0 x1874672572646912/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829960 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829935.490735:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829935.490739:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787829935.490743:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8f0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572646912:192.168.203.129@tcp:38:kworker.0 00000100:00100000:0.0:1787829935.490768:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8fc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572647040:192.168.203.129@tcp:38:kworker.0 00000800:00000100:0.0:1787829935.490797:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787829935.490807:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829935/real 1787829935] req@ffff8cb998a8fc00 x1874672572647040/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829960 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829935.490818:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8fc00 x1874672572647040/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829960 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829935.490834:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829935.490837:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787829935.490841:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8fc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572647040:192.168.203.129@tcp:38:kworker.0 00010000:00010000:2.0:1787829936.322414:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787829937.346416:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787829938.370449:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829939.394429:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787829940.418412:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787829940.610367:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829940.610378:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787829940.610382:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829940.610391:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 2892 00000100:00080000:2.0:1787829940.610402:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829940.610409:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:2.0:1787829940.610441:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8e1c0 x1874672572647168/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829940.610467:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829940.610471:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787829940.610473:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829940.610477:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 2892 00000100:00080000:2.0:1787829940.610481:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829940.610485:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:2.0:1787829940.610495:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787829940.610509:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8e1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572647168:192.168.203.129@tcp:250:kworker.0 00000100:00100000:2.0:1787829940.610509:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8da40 x1874672572647296/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829940.610531:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829940.610535:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1787829940.610537:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829940.610541:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 2892 00000100:00080000:2.0:1787829940.610558:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829940.610564:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000800:00000100:0.0:1787829940.610565:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000008:00100000:2.0:1787829940.610573:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787829940.610587:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8de00 x1874672572647424/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787829940.610611:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829940/real 1787829940] req@ffff8cb998a8e1c0 x1874672572647168/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829965 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829940.610628:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8e1c0 x1874672572647168/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829965 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829940.610650:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829940.610654:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787829940.610658:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8e1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572647168:192.168.203.129@tcp:250:kworker.0 00000100:00100000:0.0:1787829940.610678:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8da40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572647296:192.168.203.129@tcp:38:kworker.0 00000800:00000100:0.0:1787829940.610698:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787829940.610709:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829940/real 1787829940] req@ffff8cb998a8da40 x1874672572647296/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829965 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829940.610720:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8da40 x1874672572647296/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829965 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829940.610728:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829940.610731:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787829940.610734:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8da40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572647296:192.168.203.129@tcp:38:kworker.0 00000100:00100000:0.0:1787829940.610739:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8de00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572647424:192.168.203.129@tcp:38:kworker.0 00000800:00000100:0.0:1787829940.610754:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787829940.610762:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829940/real 1787829940] req@ffff8cb998a8de00 x1874672572647424/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829965 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829940.610773:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8de00 x1874672572647424/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829965 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829940.610780:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829940.610783:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787829940.610786:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8de00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572647424:192.168.203.129@tcp:38:kworker.0 00010000:00010000:0.0:1787829941.442537:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829942.466419:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787829943.490467:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787829944.514470:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829945.538385:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787829945.730327:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829945.730336:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787829945.730340:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829945.730350:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 2897 00000100:00080000:2.0:1787829945.730360:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829945.730366:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:2.0:1787829945.730401:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8d2c0 x1874672572647552/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829945.730421:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829945.730424:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787829945.730425:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829945.730427:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 2897 00000100:00080000:2.0:1787829945.730430:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829945.730432:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:2.0:1787829945.730441:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787829945.730450:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8c3c0 x1874672572647680/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787829945.730460:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8d2c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572647552:192.168.203.129@tcp:250:kworker.0 00000100:00080000:2.0:1787829945.730477:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829945.730480:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1787829945.730481:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829945.730484:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 2897 00000100:00080000:2.0:1787829945.730487:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829945.730489:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:2.0:1787829945.730492:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787829945.730502:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8de00 x1874672572647808/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1787829945.730534:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787829945.730569:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829945/real 1787829945] req@ffff8cb998a8d2c0 x1874672572647552/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829970 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829945.730588:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8d2c0 x1874672572647552/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829970 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829945.730597:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829945.730599:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787829945.730602:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8d2c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572647552:192.168.203.129@tcp:250:kworker.0 00000100:00100000:0.0:1787829945.730622:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8c3c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572647680:192.168.203.129@tcp:38:kworker.0 00000800:00000100:0.0:1787829945.730636:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787829945.730643:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829945/real 1787829945] req@ffff8cb998a8c3c0 x1874672572647680/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829970 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829945.730650:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8c3c0 x1874672572647680/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829970 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829945.730655:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829945.730657:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787829945.730659:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8c3c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572647680:192.168.203.129@tcp:38:kworker.0 00000100:00100000:0.0:1787829945.730662:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8de00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572647808:192.168.203.129@tcp:38:kworker.0 00000800:00000100:0.0:1787829945.730680:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787829945.730686:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829945/real 1787829945] req@ffff8cb998a8de00 x1874672572647808/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829970 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829945.730693:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8de00 x1874672572647808/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829970 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829945.730697:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829945.730699:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787829945.730701:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8de00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572647808:192.168.203.129@tcp:38:kworker.0 00010000:00010000:0.0:1787829946.562442:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787829947.586393:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829948.610412:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787829949.634447:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787829950.658407:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787829950.850410:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829950.850420:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787829950.850424:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829950.850435:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 2903 00000100:00080000:0.0:1787829950.850447:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829950.850454:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:0.0:1787829950.850503:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c6b840 x1874672572647936/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829950.850541:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829950.850544:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787829950.850546:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829950.850549:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 2903 00000100:00080000:0.0:1787829950.850571:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1787829950.850577:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c6b840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572647936:192.168.203.129@tcp:250:kworker.0 00000100:00080000:0.0:1787829950.850578:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:0.0:1787829950.850592:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787829950.850609:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c69a40 x1874672572648064/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829950.850619:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829950.850622:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1787829950.850623:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829950.850627:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 2903 00000100:00080000:0.0:1787829950.850630:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829950.850632:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:0.0:1787829950.850635:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787829950.850644:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c68f00 x1874672572648192/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1787829950.850658:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787829950.850725:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829950/real 1787829950] req@ffff8cb999c6b840 x1874672572647936/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829975 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829950.850747:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c6b840 x1874672572647936/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829975 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829950.850760:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829950.850765:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787829950.850770:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c6b840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572647936:192.168.203.129@tcp:250:kworker.0 00000100:00100000:2.0:1787829950.850799:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c69a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572648064:192.168.203.129@tcp:38:kworker.0 00000800:00000100:2.0:1787829950.850826:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787829950.850865:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829950/real 1787829950] req@ffff8cb999c69a40 x1874672572648064/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829975 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829950.850874:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c69a40 x1874672572648064/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829975 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829950.850881:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829950.850883:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787829950.850886:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c69a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572648064:192.168.203.129@tcp:38:kworker.0 00000100:00100000:2.0:1787829950.850891:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c68f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572648192:192.168.203.129@tcp:38:kworker.0 00000800:00000100:2.0:1787829950.850907:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787829950.850914:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829950/real 1787829950] req@ffff8cb999c68f00 x1874672572648192/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829975 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829950.850922:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c68f00 x1874672572648192/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829975 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829950.850928:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829950.850930:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787829950.850932:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c68f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572648192:192.168.203.129@tcp:38:kworker.0 00010000:00010000:2.0:1787829951.682448:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787829952.706448:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787829953.730397:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829954.754397:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787829955.778480:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787829955.970407:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829955.970418:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787829955.970423:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829955.970436:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 2908 00000100:00080000:2.0:1787829955.970449:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829955.970478:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:2.0:1787829955.970518:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8da40 x1874672572648320/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829955.970546:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829955.970550:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787829955.970552:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829955.970556:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 2908 00000100:00080000:2.0:1787829955.970561:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829955.970565:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:2.0:1787829955.970576:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787829955.970579:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8da40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572648320:192.168.203.129@tcp:250:kworker.0 00000100:00100000:2.0:1787829955.970591:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8e1c0 x1874672572648448/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829955.970602:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829955.970605:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1787829955.970607:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829955.970610:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 2908 00000100:00080000:2.0:1787829955.970614:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829955.970617:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:2.0:1787829955.970622:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787829955.970633:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8fc00 x1874672572648576/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1787829955.970656:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787829955.970703:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829955/real 1787829955] req@ffff8cb998a8da40 x1874672572648320/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829980 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829955.970718:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8da40 x1874672572648320/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829980 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829955.970743:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829955.970748:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787829955.970753:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8da40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572648320:192.168.203.129@tcp:250:kworker.0 00000100:00100000:0.0:1787829955.970779:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8e1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572648448:192.168.203.129@tcp:38:kworker.0 00000800:00000100:0.0:1787829955.970804:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787829955.970817:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829955/real 1787829955] req@ffff8cb998a8e1c0 x1874672572648448/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829980 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829955.970828:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8e1c0 x1874672572648448/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829980 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829955.970834:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829955.970836:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787829955.970838:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8e1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572648448:192.168.203.129@tcp:38:kworker.0 00000100:00100000:0.0:1787829955.970842:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8fc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572648576:192.168.203.129@tcp:38:kworker.0 00000800:00000100:0.0:1787829955.970856:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787829955.970863:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829955/real 1787829955] req@ffff8cb998a8fc00 x1874672572648576/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829980 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829955.970869:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8fc00 x1874672572648576/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829980 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829955.970877:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829955.970880:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787829955.970883:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8fc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572648576:192.168.203.129@tcp:38:kworker.0 00010000:00010000:0.0:1787829956.802475:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829957.826438:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787829958.850494:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787829959.874460:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829960.898544:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787829961.090408:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829961.090420:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787829961.090425:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829961.090437:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 2913 00000100:00080000:2.0:1787829961.090451:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829961.090460:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:2.0:1787829961.090505:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8f0c0 x1874672572648704/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787829961.090583:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8f0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572648704:192.168.203.129@tcp:250:kworker.0 00000800:00000100:0.0:1787829961.090658:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00080000:2.0:1787829961.090692:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829961.090696:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787829961.090698:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829961.090703:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 2913 00000100:00080000:2.0:1787829961.090707:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829961.090712:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:2.0:1787829961.090724:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787829961.090739:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8c780 x1874672572648832/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787829961.091071:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829961/real 1787829961] req@ffff8cb998a8f0c0 x1874672572648704/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829986 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829961.091091:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8f0c0 x1874672572648704/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829986 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829961.091101:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829961.091105:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787829961.091109:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8f0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572648704:192.168.203.129@tcp:250:kworker.0 00000100:00100000:0.0:1787829961.091135:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8c780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572648832:192.168.203.129@tcp:38:kworker.0 00000800:00000100:0.0:1787829961.091159:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787829961.091172:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829961/real 1787829961] req@ffff8cb998a8c780 x1874672572648832/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829986 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829961.091182:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8c780 x1874672572648832/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829986 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829961.091190:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829961.091193:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787829961.091197:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8c780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572648832:192.168.203.129@tcp:38:kworker.0 00000100:00080000:2.0:1787829961.091287:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829961.091292:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1787829961.091294:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829961.091299:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 2913 00000100:00080000:2.0:1787829961.091303:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829961.091309:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:2.0:1787829961.091316:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787829961.091346:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8cf00 x1874672572648960/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787829961.091392:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8cf00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572648960:192.168.203.129@tcp:38:kworker.0 00000800:00000100:0.0:1787829961.091424:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787829961.091436:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829961/real 1787829961] req@ffff8cb998a8cf00 x1874672572648960/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829986 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829961.091448:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8cf00 x1874672572648960/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829986 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829961.091456:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829961.091459:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787829961.091462:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8cf00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572648960:192.168.203.129@tcp:38:kworker.0 00010000:00010000:0.0:1787829961.922517:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787829962.946434:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829963.970745:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787829964.995568:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787829966.018501:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787829966.210375:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829966.210383:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787829966.210387:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829966.210396:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 2918 00000100:00080000:0.0:1787829966.210406:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829966.210413:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:0.0:1787829966.210446:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c6bc00 x1874672572649088/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829966.210474:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829966.210477:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787829966.210479:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829966.210482:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 2918 00000100:00080000:0.0:1787829966.210501:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829966.210506:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:0.0:1787829966.210516:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787829966.210523:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c6bc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572649088:192.168.203.129@tcp:250:kworker.0 00000100:00100000:0.0:1787829966.210537:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c6b480 x1874672572649216/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829966.210552:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829966.210556:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1787829966.210558:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829966.210562:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 2918 00000100:00080000:0.0:1787829966.210566:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829966.210570:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:0.0:1787829966.210575:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787829966.210586:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c69e00 x1874672572649344/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1787829966.210593:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787829966.210635:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829966/real 1787829966] req@ffff8cb999c6bc00 x1874672572649088/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829991 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829966.210647:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c6bc00 x1874672572649088/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829991 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829966.210654:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829966.210657:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787829966.210660:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c6bc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572649088:192.168.203.129@tcp:250:kworker.0 00000100:00100000:2.0:1787829966.210679:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c6b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572649216:192.168.203.129@tcp:38:kworker.0 00000800:00000100:2.0:1787829966.210701:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787829966.210713:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829966/real 1787829966] req@ffff8cb999c6b480 x1874672572649216/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829991 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829966.210725:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c6b480 x1874672572649216/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829991 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829966.210729:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829966.210732:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787829966.210733:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c6b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572649216:192.168.203.129@tcp:38:kworker.0 00000100:00100000:2.0:1787829966.210737:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c69e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572649344:192.168.203.129@tcp:38:kworker.0 00000800:00000100:2.0:1787829966.210756:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787829966.210763:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829966/real 1787829966] req@ffff8cb999c69e00 x1874672572649344/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829991 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829966.210770:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c69e00 x1874672572649344/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829991 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829966.210775:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829966.210776:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787829966.210778:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c69e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572649344:192.168.203.129@tcp:38:kworker.0 00010000:00010000:2.0:1787829967.042412:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787829968.066871:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787829969.090500:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829970.114406:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787829971.138514:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787829971.330605:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829971.330619:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787829971.330625:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829971.330637:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 2923 00000100:00080000:1.0:1787829971.330660:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829971.330677:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:1.0:1787829971.330729:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f3b480 x1874672572649472/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829971.330766:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829971.330897:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787829971.330899:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829971.330905:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 2923 00000100:00080000:1.0:1787829971.330909:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829971.330914:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:2.0:1787829971.330922:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f3b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572649472:192.168.203.129@tcp:250:kworker.0 00000008:00100000:1.0:1787829971.330926:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787829971.330944:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f39a40 x1874672572649600/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829971.330966:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829971.330970:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1787829971.330972:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829971.330977:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 2923 00000100:00080000:1.0:1787829971.330982:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829971.330986:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:1.0:1787829971.330991:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787829971.331006:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f39e00 x1874672572649728/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1787829971.331007:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787829971.331071:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829971/real 1787829971] req@ffff8cb999f3b480 x1874672572649472/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829996 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829971.331094:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f3b480 x1874672572649472/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829996 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829971.331105:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829971.331109:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787829971.331113:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f3b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572649472:192.168.203.129@tcp:250:kworker.0 00000100:00100000:2.0:1787829971.331140:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f39a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572649600:192.168.203.129@tcp:38:kworker.0 00000800:00000100:2.0:1787829971.331165:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787829971.331178:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829971/real 1787829971] req@ffff8cb999f39a40 x1874672572649600/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829996 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829971.331197:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f39a40 x1874672572649600/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829996 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829971.331205:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829971.331208:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787829971.331211:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f39a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572649600:192.168.203.129@tcp:38:kworker.0 00000100:00100000:2.0:1787829971.331217:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f39e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572649728:192.168.203.129@tcp:38:kworker.0 00000800:00000100:2.0:1787829971.331241:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787829971.331253:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829971/real 1787829971] req@ffff8cb999f39e00 x1874672572649728/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829996 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829971.331290:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f39e00 x1874672572649728/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829996 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829971.331298:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829971.331301:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787829971.331304:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f39e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572649728:192.168.203.129@tcp:38:kworker.0 00010000:00010000:0.0:1787829972.162491:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829973.187132:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787829974.210472:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787829975.234453:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829976.259015:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787829976.450392:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829976.450406:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787829976.450412:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829976.450424:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 2928 00000100:00080000:1.0:1787829976.450436:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829976.450444:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:1.0:1787829976.450490:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f3b840 x1874672572649856/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829976.450554:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829976.450558:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787829976.450560:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829976.450565:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 2928 00000100:00080000:1.0:1787829976.450569:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829976.450574:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:1.0:1787829976.450585:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787829976.450599:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f39e00 x1874672572649984/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787829976.450603:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f3b840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572649856:192.168.203.129@tcp:250:kworker.0 00000100:00080000:1.0:1787829976.450609:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829976.450612:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1787829976.450613:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829976.450617:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 2928 00000100:00080000:1.0:1787829976.450619:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829976.450633:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:1.0:1787829976.450637:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787829976.450646:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f39a40 x1874672572650112/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787829976.450739:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f39e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572649984:192.168.203.129@tcp:38:kworker.0 00000100:00100000:2.0:1787829976.450769:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f39a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572650112:192.168.203.129@tcp:38:kworker.0 00010000:00010000:0.0:1787829977.282528:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787829978.306468:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829979.330642:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:3.0:1787829979.522586:0:2384:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.129:988 00000400:00000100:3.0:1787829979.522643:0:2384:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.129@tcp at host 192.168.203.129 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:3.0:1787829979.522663:0:2384:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.129@tcp 00000800:00000100:3.0:1787829979.522707:0:2384:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.129@tcp 00000800:00000100:3.0:1787829979.522717:0:2384:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.129@tcp 00000100:00100000:2.0:1787829979.522784:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829976/real 1787829979] req@ffff8cb999f39e00 x1874672572649984/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830001 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829979.522827:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f39e00 x1874672572649984/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830001 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829979.522840:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829979.522845:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787829979.522850:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f39e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572649984:192.168.203.129@tcp:38:kworker.0 00000100:00100000:2.0:1787829979.522858:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829976/real 1787829979] req@ffff8cb999f39a40 x1874672572650112/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830001 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829979.522869:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f39a40 x1874672572650112/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830001 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829979.522876:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829979.522880:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787829979.522882:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f39a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572650112:192.168.203.129@tcp:38:kworker.0 00000100:00100000:2.0:1787829979.522888:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829976/real 1787829979] req@ffff8cb999f3b840 x1874672572649856/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830001 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829979.522899:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f3b840 x1874672572649856/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830001 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829979.522907:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829979.522910:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787829979.522913:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f3b840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572649856:192.168.203.129@tcp:250:kworker.0 00010000:00010000:0.0:1787829980.354459:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787829981.378866:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787829981.570564:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829981.570580:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787829981.570584:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829981.570594:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 2933 00000100:00080000:2.0:1787829981.570619:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829981.570627:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:2.0:1787829981.570680:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8e580 x1874672572650240/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829981.570705:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829981.570708:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787829981.570711:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829981.570715:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 2933 00000100:00080000:2.0:1787829981.570718:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829981.570723:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:2.0:1787829981.570735:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787829981.570741:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8e580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572650240:192.168.203.129@tcp:250:kworker.0 00000100:00100000:2.0:1787829981.570748:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8d680 x1874672572650368/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829981.570756:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829981.570759:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1787829981.570761:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829981.570764:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 2933 00000100:00080000:2.0:1787829981.570767:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829981.570770:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:2.0:1787829981.570774:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787829981.570805:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8f480 x1874672572650496/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1787829981.570831:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787829981.570902:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829981/real 1787829981] req@ffff8cb998a8e580 x1874672572650240/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830006 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829981.570935:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8e580 x1874672572650240/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830006 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829981.570945:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829981.570949:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787829981.570953:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8e580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572650240:192.168.203.129@tcp:250:kworker.0 00000100:00100000:1.0:1787829981.570983:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8d680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572650368:192.168.203.129@tcp:38:kworker.0 00000800:00000100:1.0:1787829981.571010:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787829981.571029:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829981/real 1787829981] req@ffff8cb998a8d680 x1874672572650368/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830006 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829981.571040:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8d680 x1874672572650368/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830006 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829981.571048:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829981.571051:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787829981.571054:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8d680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572650368:192.168.203.129@tcp:38:kworker.0 00000100:00100000:1.0:1787829981.571060:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572650496:192.168.203.129@tcp:38:kworker.0 00000800:00000100:1.0:1787829981.571080:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787829981.571092:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829981/real 1787829981] req@ffff8cb998a8f480 x1874672572650496/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830006 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829981.571108:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8f480 x1874672572650496/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830006 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829981.571127:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829981.571131:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787829981.571134:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572650496:192.168.203.129@tcp:38:kworker.0 00010000:00010000:2.0:1787829982.402459:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787829983.426941:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787829984.450537:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829985.476200:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787829986.500685:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787829986.690406:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829986.690417:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787829986.690421:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829986.690431:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 2938 00000100:00080000:2.0:1787829986.690442:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829986.690450:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:2.0:1787829986.690487:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8c000 x1874672572650624/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829986.690521:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829986.690525:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787829986.690527:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829986.690530:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 2938 00000100:00080000:2.0:1787829986.690534:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829986.690537:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:2.0:1787829986.690546:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787829986.690559:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8cb40 x1874672572650752/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829986.690568:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829986.690571:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1787829986.690573:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829986.690576:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 2938 00000100:00080000:2.0:1787829986.690594:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829986.690598:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:2.0:1787829986.690603:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787829986.690614:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8ed00 x1874672572650880/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787829986.690712:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572650624:192.168.203.129@tcp:250:kworker.0 00000800:00000100:1.0:1787829986.690784:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787829986.690824:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829986/real 1787829986] req@ffff8cb998a8c000 x1874672572650624/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830011 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829986.690840:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8c000 x1874672572650624/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830011 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829986.690850:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829986.690854:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787829986.690857:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572650624:192.168.203.129@tcp:250:kworker.0 00000100:00100000:1.0:1787829986.690863:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8cb40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572650752:192.168.203.129@tcp:38:kworker.0 00000800:00000100:1.0:1787829986.690884:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787829986.690897:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829986/real 1787829986] req@ffff8cb998a8cb40 x1874672572650752/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830011 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829986.690908:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8cb40 x1874672572650752/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830011 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829986.690915:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829986.690917:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787829986.690920:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8cb40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572650752:192.168.203.129@tcp:38:kworker.0 00000100:00100000:1.0:1787829986.690932:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8ed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572650880:192.168.203.129@tcp:38:kworker.0 00000800:00000100:1.0:1787829986.690952:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787829986.690964:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829986/real 1787829986] req@ffff8cb998a8ed00 x1874672572650880/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830011 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829986.690975:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8ed00 x1874672572650880/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830011 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829986.690983:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829986.690986:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787829986.690989:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8ed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572650880:192.168.203.129@tcp:38:kworker.0 00010000:00010000:0.0:1787829987.525851:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829988.546563:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787829989.570571:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787829990.594861:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829991.618489:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787829991.810451:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829991.810464:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787829991.810469:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829991.810481:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 2943 00000100:00080000:2.0:1787829991.810493:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829991.810500:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:2.0:1787829991.810541:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8e940 x1874672572651008/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1787829991.810616:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8e940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572651008:192.168.203.129@tcp:250:kworker.0 00000800:00000100:3.0:1787829991.810688:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00080000:2.0:1787829991.810728:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829991.810732:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787829991.810734:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829991.810738:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 2943 00000100:00080000:2.0:1787829991.810742:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829991.810746:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:2.0:1787829991.810758:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787829991.810770:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8f840 x1874672572651136/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829991.811435:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829991.811445:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1787829991.811449:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829991.811456:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 2943 00000100:00080000:2.0:1787829991.811466:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829991.811473:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:2.0:1787829991.811484:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787829991.811505:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829991/real 1787829991] req@ffff8cb998a8e940 x1874672572651008/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830016 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829991.811517:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8ed00 x1874672572651264/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1787829991.811539:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8e940 x1874672572651008/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830016 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787829991.811551:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829991.811556:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:3.0:1787829991.811562:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8e940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572651008:192.168.203.129@tcp:250:kworker.0 00000100:00100000:3.0:1787829991.811589:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8f840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572651136:192.168.203.129@tcp:38:kworker.0 00000800:00000100:3.0:1787829991.811616:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:3.0:1787829991.811629:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829991/real 1787829991] req@ffff8cb998a8f840 x1874672572651136/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830016 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787829991.811640:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8f840 x1874672572651136/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830016 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787829991.811647:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829991.811657:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:3.0:1787829991.811660:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8f840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572651136:192.168.203.129@tcp:38:kworker.0 00000100:00100000:3.0:1787829991.811667:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8ed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572651264:192.168.203.129@tcp:38:kworker.0 00000800:00000100:3.0:1787829991.811685:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:3.0:1787829991.811695:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829991/real 1787829991] req@ffff8cb998a8ed00 x1874672572651264/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830016 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787829991.811705:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8ed00 x1874672572651264/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830016 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787829991.811726:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829991.811730:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:3.0:1787829991.811733:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8ed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572651264:192.168.203.129@tcp:38:kworker.0 00010000:00010000:0.0:1787829992.642535:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787829993.666462:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829994.690412:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787829995.714445:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787829996.738389:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787829996.930360:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829996.930371:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787829996.930376:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829996.930385:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 2949 00000100:00080000:0.0:1787829996.930398:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829996.930407:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:0.0:1787829996.930448:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c692c0 x1874672572651392/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829996.930473:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829996.930477:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787829996.930478:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829996.930481:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 2949 00000100:00080000:0.0:1787829996.930485:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829996.930488:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:0.0:1787829996.930499:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787829996.930504:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c692c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572651392:192.168.203.129@tcp:250:kworker.0 00000100:00100000:0.0:1787829996.930513:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c69680 x1874672572651520/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829996.930522:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829996.930531:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1787829996.930532:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829996.930536:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 2949 00000100:00080000:0.0:1787829996.930540:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829996.930543:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:0.0:1787829996.930547:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787829996.930557:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c6a1c0 x1874672572651648/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1787829996.930569:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787829996.930606:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829996/real 1787829996] req@ffff8cb999c692c0 x1874672572651392/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830021 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829996.930619:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c692c0 x1874672572651392/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830021 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829996.930627:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829996.930630:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787829996.930632:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c692c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572651392:192.168.203.129@tcp:250:kworker.0 00000100:00100000:1.0:1787829996.930654:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c69680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572651520:192.168.203.129@tcp:38:kworker.0 00000800:00000100:1.0:1787829996.930667:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787829996.930690:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829996/real 1787829996] req@ffff8cb999c69680 x1874672572651520/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830021 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829996.930705:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c69680 x1874672572651520/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830021 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829996.930711:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829996.930714:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787829996.930716:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c69680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572651520:192.168.203.129@tcp:38:kworker.0 00000100:00100000:1.0:1787829996.930720:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c6a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572651648:192.168.203.129@tcp:38:kworker.0 00000800:00000100:1.0:1787829996.930735:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787829996.930742:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829996/real 1787829996] req@ffff8cb999c6a1c0 x1874672572651648/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830021 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829996.930749:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c6a1c0 x1874672572651648/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830021 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829996.930754:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829996.930756:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787829996.930758:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c6a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572651648:192.168.203.129@tcp:38:kworker.0 00010000:00010000:2.0:1787829997.762408:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787829998.786447:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787829999.810461:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787830000.834523:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787830001.858570:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787830002.050421:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787830002.050434:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787830002.050439:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787830002.050451:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 2954 00000100:00080000:2.0:1787830002.050463:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787830002.050471:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:2.0:1787830002.050519:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8cb40 x1874672572651776/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787830002.050553:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787830002.050557:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787830002.050559:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787830002.050580:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 2954 00000100:00080000:2.0:1787830002.050585:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787830002.050596:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:1.0:1787830002.050606:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8cb40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572651776:192.168.203.129@tcp:250:kworker.0 00000008:00100000:2.0:1787830002.050611:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787830002.050629:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8c000 x1874672572651904/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787830002.050652:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787830002.050655:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1787830002.050657:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787830002.050661:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 2954 00000100:00080000:2.0:1787830002.050665:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787830002.050669:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:2.0:1787830002.050675:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787830002.050687:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8f480 x1874672572652032/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1787830002.050702:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787830002.050759:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830002/real 1787830002] req@ffff8cb998a8cb40 x1874672572651776/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830027 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830002.050778:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8cb40 x1874672572651776/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830027 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830002.050806:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830002.050811:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787830002.050816:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8cb40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572651776:192.168.203.129@tcp:250:kworker.0 00000100:00100000:1.0:1787830002.050844:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572651904:192.168.203.129@tcp:38:kworker.0 00000800:00000100:1.0:1787830002.050863:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787830002.050873:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830002/real 1787830002] req@ffff8cb998a8c000 x1874672572651904/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830027 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830002.050881:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8c000 x1874672572651904/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830027 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830002.050886:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830002.050889:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787830002.050891:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572651904:192.168.203.129@tcp:38:kworker.0 00000100:00100000:1.0:1787830002.050894:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572652032:192.168.203.129@tcp:38:kworker.0 00000800:00000100:1.0:1787830002.050905:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787830002.050911:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830002/real 1787830002] req@ffff8cb998a8f480 x1874672572652032/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830027 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830002.050919:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8f480 x1874672572652032/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830027 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830002.050927:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830002.050930:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787830002.050933:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572652032:192.168.203.129@tcp:38:kworker.0 00010000:00010000:0.0:1787830002.882452:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787830003.906476:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787830004.930505:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787830005.954885:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787830006.979043:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787830007.171499:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787830007.171510:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787830007.171515:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787830007.171527:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 2959 00000100:00080000:0.0:1787830007.171540:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787830007.171548:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:0.0:1787830007.171594:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c6a580 x1874672572652160/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787830007.171641:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787830007.171645:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787830007.171646:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787830007.171667:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 2959 00000100:00080000:0.0:1787830007.171671:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787830007.171675:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:0.0:1787830007.171685:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787830007.171697:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c68000 x1874672572652288/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787830007.171707:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787830007.171710:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1787830007.171712:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787830007.171715:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 2959 00000100:00080000:0.0:1787830007.171717:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787830007.171721:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:0.0:1787830007.171725:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787830007.171735:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c6a1c0 x1874672572652416/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787830007.171984:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c6a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572652160:192.168.203.129@tcp:250:kworker.0 00000800:00000100:1.0:1787830007.172051:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787830007.172198:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830007/real 1787830007] req@ffff8cb999c6a580 x1874672572652160/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830032 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830007.172218:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c6a580 x1874672572652160/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830032 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830007.172230:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830007.172234:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787830007.172238:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c6a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572652160:192.168.203.129@tcp:250:kworker.0 00000100:00100000:1.0:1787830007.172246:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c68000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572652288:192.168.203.129@tcp:38:kworker.0 00000800:00000100:1.0:1787830007.172299:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787830007.172313:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830007/real 1787830007] req@ffff8cb999c68000 x1874672572652288/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830032 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830007.172325:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c68000 x1874672572652288/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830032 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830007.172955:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830007.172959:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787830007.172964:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c68000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572652288:192.168.203.129@tcp:38:kworker.0 00000100:00100000:1.0:1787830007.172970:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c6a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572652416:192.168.203.129@tcp:38:kworker.0 00000800:00000100:1.0:1787830007.172994:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787830007.173006:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830007/real 1787830007] req@ffff8cb999c6a1c0 x1874672572652416/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830032 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830007.173028:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c6a1c0 x1874672572652416/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830032 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830007.173037:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830007.173040:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787830007.173043:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c6a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572652416:192.168.203.129@tcp:38:kworker.0 00010000:00010000:0.0:1787830008.003851:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787830009.026481:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787830010.050472:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787830011.074553:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787830012.099617:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787830012.291945:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787830012.291957:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787830012.291962:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787830012.291973:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 2964 00000100:00080000:0.0:1787830012.291987:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787830012.291996:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:0.0:1787830012.292062:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c69680 x1874672572652544/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787830012.292095:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787830012.292099:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787830012.292101:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787830012.292105:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 2964 00000100:00080000:0.0:1787830012.292109:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787830012.292114:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:0.0:1787830012.292125:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787830012.292137:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c692c0 x1874672572652672/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787830012.292147:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787830012.292150:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1787830012.292152:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787830012.292156:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 2964 00000100:00080000:0.0:1787830012.292159:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787830012.292163:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:0.0:1787830012.292167:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787830012.292178:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c69e00 x1874672572652800/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787830012.292258:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c69680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572652544:192.168.203.129@tcp:250:kworker.0 00000800:00000100:1.0:1787830012.292428:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787830012.292474:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830012/real 1787830012] req@ffff8cb999c69680 x1874672572652544/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830037 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830012.292487:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c69680 x1874672572652544/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830037 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830012.292495:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830012.292498:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787830012.292500:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c69680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572652544:192.168.203.129@tcp:250:kworker.0 00000100:00100000:1.0:1787830012.292505:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c692c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572652672:192.168.203.129@tcp:38:kworker.0 00000800:00000100:1.0:1787830012.292519:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787830012.292527:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830012/real 1787830012] req@ffff8cb999c692c0 x1874672572652672/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830037 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830012.292534:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c692c0 x1874672572652672/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830037 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830012.292553:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830012.292556:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787830012.292558:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c692c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572652672:192.168.203.129@tcp:38:kworker.0 00000100:00100000:1.0:1787830012.292755:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c69e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572652800:192.168.203.129@tcp:38:kworker.0 00000800:00000100:1.0:1787830012.292780:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787830012.292901:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830012/real 1787830012] req@ffff8cb999c69e00 x1874672572652800/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830037 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830012.292914:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c69e00 x1874672572652800/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830037 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830012.292923:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830012.292927:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787830012.292930:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c69e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572652800:192.168.203.129@tcp:38:kworker.0 00010000:00010000:2.0:1787830013.122753:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787830014.147108:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787830015.170890:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787830016.194515:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787830017.218464:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787830017.410585:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787830017.410942:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787830017.410946:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787830017.410958:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 2969 00000100:00080000:0.0:1787830017.410991:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787830017.411143:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:0.0:1787830017.411248:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c6b480 x1874672572652928/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787830017.411298:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787830017.411302:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787830017.411303:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787830017.411307:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 2969 00000100:00080000:0.0:1787830017.411311:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787830017.411316:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:1.0:1787830017.411368:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c6b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572652928:192.168.203.129@tcp:250:kworker.0 00000008:00100000:0.0:1787830017.411400:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787830017.411420:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c6bc00 x1874672572653056/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787830017.411432:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787830017.411436:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1787830017.411438:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787830017.411441:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 2969 00000100:00080000:0.0:1787830017.411446:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000800:00000100:1.0:1787830017.411447:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00080000:0.0:1787830017.411453:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:0.0:1787830017.411459:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787830017.411470:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c68f00 x1874672572653184/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787830017.411500:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830017/real 1787830017] req@ffff8cb999c6b480 x1874672572652928/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830042 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830017.411545:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c6b480 x1874672572652928/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830042 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830017.411560:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830017.411564:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787830017.411582:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c6b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572652928:192.168.203.129@tcp:250:kworker.0 00000100:00100000:1.0:1787830017.411619:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c6bc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572653056:192.168.203.129@tcp:38:kworker.0 00000800:00000100:1.0:1787830017.411642:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787830017.411655:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830017/real 1787830017] req@ffff8cb999c6bc00 x1874672572653056/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830042 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830017.411666:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c6bc00 x1874672572653056/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830042 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830017.411673:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830017.411677:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787830017.411680:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c6bc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572653056:192.168.203.129@tcp:38:kworker.0 00000100:00100000:1.0:1787830017.411698:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c68f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572653184:192.168.203.129@tcp:38:kworker.0 00000800:00000100:1.0:1787830017.411715:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787830017.411725:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830017/real 1787830017] req@ffff8cb999c68f00 x1874672572653184/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830042 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830017.411737:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c68f00 x1874672572653184/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830042 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830017.411745:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830017.411749:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787830017.411752:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c68f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572653184:192.168.203.129@tcp:38:kworker.0 00010000:00010000:0.0:1787830018.242488:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787830019.266704:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787830020.290524:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787830021.314543:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787830022.338670:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787830022.531650:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787830022.531664:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787830022.531670:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787830022.531704:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 2974 00000100:00080000:2.0:1787830022.531717:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787830022.531725:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:2.0:1787830022.531766:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8d680 x1874672572653312/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787830022.531802:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787830022.531806:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787830022.531808:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787830022.531812:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 2974 00000100:00080000:2.0:1787830022.531815:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787830022.531819:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:2.0:1787830022.531830:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787830022.531842:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8e580 x1874672572653440/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787830022.531852:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787830022.531855:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1787830022.531857:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787830022.531861:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 2974 00000100:00080000:2.0:1787830022.531864:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787830022.531867:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:2.0:1787830022.531872:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787830022.531880:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8cf00 x1874672572653568/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787830022.531982:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8d680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572653312:192.168.203.129@tcp:250:kworker.0 00000800:00000100:1.0:1787830022.532051:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787830022.532104:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830022/real 1787830022] req@ffff8cb998a8d680 x1874672572653312/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830047 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830022.532121:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8d680 x1874672572653312/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830047 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830022.532132:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830022.532136:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787830022.532140:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8d680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572653312:192.168.203.129@tcp:250:kworker.0 00000100:00100000:1.0:1787830022.532147:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8e580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572653440:192.168.203.129@tcp:38:kworker.0 00000800:00000100:1.0:1787830022.532167:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787830022.532178:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830022/real 1787830022] req@ffff8cb998a8e580 x1874672572653440/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830047 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830022.532189:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8e580 x1874672572653440/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830047 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830022.532197:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830022.532200:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787830022.532203:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8e580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572653440:192.168.203.129@tcp:38:kworker.0 00000100:00100000:1.0:1787830022.532209:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8cf00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572653568:192.168.203.129@tcp:38:kworker.0 00000800:00000100:1.0:1787830022.532223:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787830022.532232:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830022/real 1787830022] req@ffff8cb998a8cf00 x1874672572653568/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830047 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830022.532243:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8cf00 x1874672572653568/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830047 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830022.532291:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830022.532295:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787830022.532299:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8cf00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572653568:192.168.203.129@tcp:38:kworker.0 00010000:00010000:0.0:1787830023.362586:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787830024.388587:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787830025.410611:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787830026.434684:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787830027.458754:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787830027.650435:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787830027.650445:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787830027.650449:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787830027.650475:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 2979 00000100:00080000:3.0:1787830027.650486:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787830027.650494:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:3.0:1787830027.650529:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb9923161c0 x1874672572653696/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787830027.650554:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787830027.650557:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787830027.650559:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787830027.650562:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 2979 00000100:00080000:3.0:1787830027.650579:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787830027.650585:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:3.0:1787830027.650593:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787830027.650610:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb9923170c0 x1874672572653824/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787830027.650623:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787830027.650626:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1787830027.650627:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787830027.650637:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 2979 00000100:00100000:1.0:1787830027.650640:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb9923161c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572653696:192.168.203.129@tcp:250:kworker.0 00000100:00080000:3.0:1787830027.650644:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787830027.650655:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:3.0:1787830027.650660:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787830027.650674:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb992315a40 x1874672572653952/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1787830027.650732:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787830027.650799:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830027/real 1787830027] req@ffff8cb9923161c0 x1874672572653696/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830052 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830027.650818:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb9923161c0 x1874672572653696/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830052 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830027.650830:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830027.650835:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787830027.650839:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb9923161c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572653696:192.168.203.129@tcp:250:kworker.0 00000100:00100000:1.0:1787830027.650867:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb9923170c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572653824:192.168.203.129@tcp:38:kworker.0 00000800:00000100:1.0:1787830027.650888:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787830027.650900:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830027/real 1787830027] req@ffff8cb9923170c0 x1874672572653824/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830052 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830027.650911:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb9923170c0 x1874672572653824/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830052 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830027.650919:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830027.650923:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787830027.650926:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb9923170c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572653824:192.168.203.129@tcp:38:kworker.0 00000100:00100000:1.0:1787830027.650944:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb992315a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572653952:192.168.203.129@tcp:38:kworker.0 00000800:00000100:1.0:1787830027.650964:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787830027.650976:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830027/real 1787830027] req@ffff8cb992315a40 x1874672572653952/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830052 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830027.650989:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb992315a40 x1874672572653952/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830052 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830027.650997:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830027.651000:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787830027.651003:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb992315a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572653952:192.168.203.129@tcp:38:kworker.0 00010000:00010000:2.0:1787830028.482501:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787830029.506688:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787830030.531247:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787830031.556742:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787830032.578471:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787830032.770329:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787830032.770338:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787830032.770342:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787830032.770352:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 2984 00000100:00080000:3.0:1787830032.770361:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787830032.770369:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:3.0:1787830032.770400:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb992316940 x1874672572654080/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787830032.770443:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787830032.770462:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787830032.770464:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787830032.770468:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 2984 00000100:00080000:3.0:1787830032.770473:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787830032.770477:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:3.0:1787830032.770487:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787830032.770502:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb992315e00 x1874672572654208/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787830032.770513:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787830032.770517:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1787830032.770518:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787830032.770523:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 2984 00000100:00080000:3.0:1787830032.770525:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787830032.770529:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:3.0:1787830032.770533:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787830032.770540:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb992314f00 x1874672572654336/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787830032.770665:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb992316940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572654080:192.168.203.129@tcp:250:kworker.0 00000800:00000100:0.0:1787830032.770720:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787830032.770751:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830032/real 1787830032] req@ffff8cb992316940 x1874672572654080/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830057 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830032.770766:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb992316940 x1874672572654080/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830057 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830032.770775:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830032.770790:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787830032.770794:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb992316940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572654080:192.168.203.129@tcp:250:kworker.0 00000100:00100000:0.0:1787830032.770802:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb992315e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572654208:192.168.203.129@tcp:38:kworker.0 00000800:00000100:0.0:1787830032.770821:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787830032.770833:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830032/real 1787830032] req@ffff8cb992315e00 x1874672572654208/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830057 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830032.770843:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb992315e00 x1874672572654208/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830057 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830032.770858:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830032.770861:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787830032.770865:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb992315e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572654208:192.168.203.129@tcp:38:kworker.0 00000100:00100000:0.0:1787830032.770871:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb992314f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572654336:192.168.203.129@tcp:38:kworker.0 00000800:00000100:0.0:1787830032.770890:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787830032.770914:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830032/real 1787830032] req@ffff8cb992314f00 x1874672572654336/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830057 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830032.770936:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb992314f00 x1874672572654336/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830057 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830032.770944:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830032.770947:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787830032.770950:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb992314f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572654336:192.168.203.129@tcp:38:kworker.0 00010000:00010000:0.0:1787830033.602531:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787830034.627582:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787830035.650488:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787830036.674592:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787830037.700046:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787830037.890409:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787830037.890418:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787830037.890422:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787830037.890429:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 2990 00000100:00080000:2.0:1787830037.890437:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787830037.890443:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:2.0:1787830037.890472:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8c780 x1874672572654464/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787830037.890490:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787830037.890492:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787830037.890494:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787830037.890496:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 2990 00000100:00080000:2.0:1787830037.890498:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787830037.890500:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:2.0:1787830037.890507:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787830037.890516:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8f0c0 x1874672572654592/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787830037.890522:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8c780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572654464:192.168.203.129@tcp:250:kworker.0 00000100:00080000:2.0:1787830037.890538:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787830037.890542:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1787830037.890557:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787830037.890562:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 2990 00000100:00080000:2.0:1787830037.890567:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787830037.890572:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000800:00000100:0.0:1787830037.890577:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000008:00100000:2.0:1787830037.890580:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787830037.890595:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8fc00 x1874672572654720/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787830037.890621:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830037/real 1787830037] req@ffff8cb998a8c780 x1874672572654464/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830062 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830037.890635:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8c780 x1874672572654464/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830062 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830037.890650:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830037.890653:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787830037.890656:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8c780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572654464:192.168.203.129@tcp:250:kworker.0 00000100:00100000:0.0:1787830037.890674:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8f0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572654592:192.168.203.129@tcp:38:kworker.0 00000800:00000100:0.0:1787830037.890690:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787830037.890698:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830037/real 1787830037] req@ffff8cb998a8f0c0 x1874672572654592/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830062 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830037.890705:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8f0c0 x1874672572654592/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830062 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830037.890710:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830037.890712:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787830037.890714:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8f0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572654592:192.168.203.129@tcp:38:kworker.0 00000100:00100000:0.0:1787830037.890717:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8fc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572654720:192.168.203.129@tcp:38:kworker.0 00000800:00000100:0.0:1787830037.890726:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787830037.890731:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830037/real 1787830037] req@ffff8cb998a8fc00 x1874672572654720/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830062 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830037.890737:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8fc00 x1874672572654720/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830062 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830037.890742:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830037.890751:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787830037.890754:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8fc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572654720:192.168.203.129@tcp:38:kworker.0 00010000:00010000:0.0:1787830038.722437:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787830039.746484:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787830040.770460:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787830041.797662:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787830042.818814:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787830043.010495:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787830043.010509:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787830043.010514:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787830043.010527:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 2995 00000100:00080000:2.0:1787830043.010539:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787830043.010547:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:2.0:1787830043.010591:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8e1c0 x1874672572654848/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787830043.010628:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787830043.010632:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787830043.010633:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787830043.010637:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 2995 00000100:00080000:2.0:1787830043.010641:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787830043.010644:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:2.0:1787830043.010656:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787830043.010668:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8da40 x1874672572654976/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787830043.010695:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787830043.010698:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1787830043.010700:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787830043.010704:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 2995 00000100:00080000:2.0:1787830043.010707:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787830043.010711:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:2.0:1787830043.010716:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787830043.010727:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8de00 x1874672572655104/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787830043.010950:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8e1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572654848:192.168.203.129@tcp:250:kworker.0 00000100:00100000:0.0:1787830043.011239:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8da40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572654976:192.168.203.129@tcp:38:kworker.0 00000100:00100000:0.0:1787830043.011306:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8de00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572655104:192.168.203.129@tcp:38:kworker.0 00010000:00010000:2.0:1787830043.842461:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787830044.866455:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787830045.890700:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:3.0:1787830046.082658:0:2381:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.129:988 00000400:00000100:3.0:1787830046.082705:0:2381:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.129@tcp at host 192.168.203.129 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:3.0:1787830046.082721:0:2381:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.129@tcp 00000800:00000100:3.0:1787830046.082756:0:2381:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.129@tcp 00000800:00000100:3.0:1787830046.082762:0:2381:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.129@tcp 00000100:00100000:0.0:1787830046.082803:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830043/real 1787830046] req@ffff8cb998a8e1c0 x1874672572654848/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830068 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830046.082823:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8e1c0 x1874672572654848/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830068 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830046.082832:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830046.082836:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787830046.082839:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8e1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572654848:192.168.203.129@tcp:250:kworker.0 00000100:00100000:0.0:1787830046.082845:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830043/real 1787830046] req@ffff8cb998a8da40 x1874672572654976/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830068 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830046.082852:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8da40 x1874672572654976/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830068 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830046.082856:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830046.082858:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787830046.082877:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8da40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572654976:192.168.203.129@tcp:38:kworker.0 00000100:00100000:0.0:1787830046.082885:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830043/real 1787830046] req@ffff8cb998a8de00 x1874672572655104/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830068 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830046.082897:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8de00 x1874672572655104/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830068 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830046.082906:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830046.082910:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787830046.082913:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8de00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572655104:192.168.203.129@tcp:38:kworker.0 00010000:00010000:0.0:1787830046.914577:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787830047.938648:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787830048.130400:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787830048.130412:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787830048.130417:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787830048.130429:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 3000 00000100:00080000:2.0:1787830048.130441:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787830048.130449:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:2.0:1787830048.130726:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8c3c0 x1874672572655232/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787830048.131471:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787830048.131478:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787830048.131481:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787830048.131487:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 3000 00000100:00080000:2.0:1787830048.131496:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787830048.131526:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:2.0:1787830048.131539:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787830048.131558:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8d2c0 x1874672572655360/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787830048.131569:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787830048.131572:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1787830048.131574:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787830048.131577:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 3000 00000100:00080000:2.0:1787830048.131581:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787830048.131585:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:2.0:1787830048.131600:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787830048.131612:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8de00 x1874672572655488/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787830048.131738:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8c3c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572655232:192.168.203.129@tcp:250:kworker.0 00000800:00000100:0.0:1787830048.131817:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787830048.131869:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830048/real 1787830048] req@ffff8cb998a8c3c0 x1874672572655232/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830073 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830048.131887:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8c3c0 x1874672572655232/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830073 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830048.131899:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830048.131910:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787830048.131914:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8c3c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572655232:192.168.203.129@tcp:250:kworker.0 00000100:00100000:0.0:1787830048.131921:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8d2c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572655360:192.168.203.129@tcp:38:kworker.0 00000800:00000100:0.0:1787830048.131945:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787830048.131958:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830048/real 1787830048] req@ffff8cb998a8d2c0 x1874672572655360/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830073 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830048.131978:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8d2c0 x1874672572655360/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830073 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830048.131986:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830048.131989:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787830048.131992:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8d2c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572655360:192.168.203.129@tcp:38:kworker.0 00000100:00100000:0.0:1787830048.131998:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8de00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572655488:192.168.203.129@tcp:38:kworker.0 00000800:00000100:0.0:1787830048.132018:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787830048.132029:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830048/real 1787830048] req@ffff8cb998a8de00 x1874672572655488/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830073 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830048.132040:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8de00 x1874672572655488/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830073 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830048.132047:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830048.132050:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787830048.132052:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8de00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572655488:192.168.203.129@tcp:38:kworker.0 00010000:00010000:0.0:1787830048.962762:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787830049.988370:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787830051.011362:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787830052.034740:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787830053.058508:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787830053.250397:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787830053.250404:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787830053.250407:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787830053.250416:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 3005 00000100:00080000:1.0:1787830053.250425:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787830053.250431:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:1.0:1787830053.250461:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f3b480 x1874672572655616/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787830053.250481:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787830053.250483:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787830053.250485:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787830053.250487:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 3005 00000100:00080000:1.0:1787830053.250489:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787830053.250504:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:1.0:1787830053.250511:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787830053.250520:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f3b0c0 x1874672572655744/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787830053.250528:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787830053.250530:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1787830053.250531:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787830053.250536:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 3005 00000100:00100000:0.0:1787830053.250537:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f3b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572655616:192.168.203.129@tcp:250:kworker.0 00000100:00080000:1.0:1787830053.250539:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787830053.250544:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:1.0:1787830053.250548:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787830053.250554:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f38b40 x1874672572655872/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1787830053.250611:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787830053.250652:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830053/real 1787830053] req@ffff8cb999f3b480 x1874672572655616/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830078 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830053.250677:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f3b480 x1874672572655616/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830078 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830053.250687:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830053.250692:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787830053.250695:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f3b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572655616:192.168.203.129@tcp:250:kworker.0 00000100:00100000:0.0:1787830053.250719:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f3b0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572655744:192.168.203.129@tcp:38:kworker.0 00000800:00000100:0.0:1787830053.250741:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787830053.250753:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830053/real 1787830053] req@ffff8cb999f3b0c0 x1874672572655744/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830078 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830053.250765:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f3b0c0 x1874672572655744/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830078 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830053.250774:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830053.250777:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787830053.250780:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f3b0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572655744:192.168.203.129@tcp:38:kworker.0 00000100:00100000:0.0:1787830053.250786:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f38b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572655872:192.168.203.129@tcp:38:kworker.0 00000800:00000100:0.0:1787830053.250804:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787830053.250815:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830053/real 1787830053] req@ffff8cb999f38b40 x1874672572655872/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830078 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830053.250828:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f38b40 x1874672572655872/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830078 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830053.250852:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830053.250856:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787830053.250859:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f38b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572655872:192.168.203.129@tcp:38:kworker.0 00010000:00010000:0.0:1787830054.082447:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787830055.113362:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787830056.130603:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787830057.156583:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787830058.178436:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787830058.370606:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787830058.370616:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787830058.370622:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787830058.370631:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 3010 00000100:00080000:2.0:1787830058.370641:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787830058.370648:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:2.0:1787830058.370681:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8da40 x1874672572656000/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787830058.370710:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787830058.370714:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787830058.370728:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787830058.370733:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 3010 00000100:00080000:2.0:1787830058.370737:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787830058.370741:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:2.0:1787830058.370750:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787830058.370761:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8e1c0 x1874672572656128/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787830058.370770:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787830058.370773:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1787830058.370774:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787830058.370778:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 3010 00000100:00080000:2.0:1787830058.370780:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787830058.370783:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:2.0:1787830058.370787:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787830058.370795:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8fc00 x1874672572656256/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787830058.371175:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8da40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572656000:192.168.203.129@tcp:250:kworker.0 00000800:00000100:0.0:1787830058.371290:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787830058.371337:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830058/real 1787830058] req@ffff8cb998a8da40 x1874672572656000/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830083 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830058.371353:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8da40 x1874672572656000/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830083 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830058.371363:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830058.371367:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787830058.371371:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8da40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572656000:192.168.203.129@tcp:250:kworker.0 00000100:00100000:0.0:1787830058.371378:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8e1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572656128:192.168.203.129@tcp:38:kworker.0 00000800:00000100:0.0:1787830058.371409:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787830058.371421:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830058/real 1787830058] req@ffff8cb998a8e1c0 x1874672572656128/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830083 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830058.371432:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8e1c0 x1874672572656128/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830083 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830058.371447:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830058.371450:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787830058.371454:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8e1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572656128:192.168.203.129@tcp:38:kworker.0 00000100:00100000:0.0:1787830058.371460:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8fc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572656256:192.168.203.129@tcp:38:kworker.0 00000800:00000100:0.0:1787830058.371479:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787830058.371490:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830058/real 1787830058] req@ffff8cb998a8fc00 x1874672572656256/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830083 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830058.371501:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8fc00 x1874672572656256/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830083 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830058.371508:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830058.371512:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787830058.371515:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8fc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572656256:192.168.203.129@tcp:38:kworker.0 00010000:00010000:0.0:1787830059.202541:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787830060.227418:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787830061.250599:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787830062.274459:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787830063.298995:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787830063.490978:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787830063.490987:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787830063.490992:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787830063.491001:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 3015 00000100:00080000:1.0:1787830063.491011:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787830063.491019:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:1.0:1787830063.491056:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f38f00 x1874672572656384/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787830063.491086:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787830063.491090:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787830063.491104:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787830063.491109:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 3015 00000100:00080000:1.0:1787830063.491113:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787830063.491117:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:1.0:1787830063.491127:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787830063.491137:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f392c0 x1874672572656512/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787830063.491147:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787830063.491150:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1787830063.491151:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787830063.491155:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 3015 00000100:00080000:1.0:1787830063.491157:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787830063.491161:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:1.0:1787830063.491165:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787830063.491172:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f3a580 x1874672572656640/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787830063.491258:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f38f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572656384:192.168.203.129@tcp:250:kworker.0 00000800:00000100:0.0:1787830063.491339:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787830063.491459:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830063/real 1787830063] req@ffff8cb999f38f00 x1874672572656384/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830088 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830063.491476:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f38f00 x1874672572656384/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830088 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830063.491491:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830063.491495:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787830063.491499:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f38f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572656384:192.168.203.129@tcp:250:kworker.0 00000100:00100000:0.0:1787830063.491506:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f392c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572656512:192.168.203.129@tcp:38:kworker.0 00000800:00000100:0.0:1787830063.491526:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787830063.491537:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830063/real 1787830063] req@ffff8cb999f392c0 x1874672572656512/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830088 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830063.491548:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f392c0 x1874672572656512/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830088 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830063.491555:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830063.491558:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787830063.491561:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f392c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572656512:192.168.203.129@tcp:38:kworker.0 00000100:00100000:0.0:1787830063.491567:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f3a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572656640:192.168.203.129@tcp:38:kworker.0 00000800:00000100:0.0:1787830063.491581:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787830063.491643:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830063/real 1787830063] req@ffff8cb999f3a580 x1874672572656640/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830088 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830063.491655:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f3a580 x1874672572656640/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830088 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830063.491662:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830063.491665:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787830063.491668:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f3a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572656640:192.168.203.129@tcp:38:kworker.0 00010000:00010000:0.0:1787830064.322497:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787830065.347128:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787830066.374849:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787830067.394689:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787830068.419105:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787830068.610654:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787830068.610662:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787830068.610666:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787830068.610675:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 3020 00000100:00080000:2.0:1787830068.610685:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787830068.610691:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:2.0:1787830068.610718:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8f0c0 x1874672572656768/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787830068.610754:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787830068.610758:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787830068.610760:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787830068.610764:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 3020 00000100:00080000:2.0:1787830068.610768:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787830068.610772:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:2.0:1787830068.610781:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787830068.610794:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8c780 x1874672572656896/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787830068.610804:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787830068.610807:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1787830068.610809:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787830068.610812:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 3020 00000100:00080000:2.0:1787830068.610816:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787830068.610819:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:2.0:1787830068.610823:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787830068.610833:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8cf00 x1874672572657024/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787830068.610900:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8f0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572656768:192.168.203.129@tcp:250:kworker.0 00000800:00000100:0.0:1787830068.610976:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787830068.611012:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830068/real 1787830068] req@ffff8cb998a8f0c0 x1874672572656768/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830093 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830068.611028:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8f0c0 x1874672572656768/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830093 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830068.611037:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830068.611041:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787830068.611045:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8f0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572656768:192.168.203.129@tcp:250:kworker.0 00000100:00100000:0.0:1787830068.611062:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8c780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572656896:192.168.203.129@tcp:38:kworker.0 00000800:00000100:0.0:1787830068.611085:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787830068.611099:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830068/real 1787830068] req@ffff8cb998a8c780 x1874672572656896/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830093 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830068.611109:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8c780 x1874672572656896/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830093 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830068.611117:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830068.611120:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787830068.611123:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8c780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572656896:192.168.203.129@tcp:38:kworker.0 00000100:00100000:0.0:1787830068.611137:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8cf00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572657024:192.168.203.129@tcp:38:kworker.0 00000800:00000100:0.0:1787830068.611159:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787830068.611170:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830068/real 1787830068] req@ffff8cb998a8cf00 x1874672572657024/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830093 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830068.611202:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8cf00 x1874672572657024/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830093 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830068.611211:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830068.611215:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787830068.611219:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8cf00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572657024:192.168.203.129@tcp:38:kworker.0 00010000:00010000:0.0:1787830069.442948:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787830070.467114:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787830071.490615:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787830072.515801:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787830073.538535:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787830073.730586:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787830073.730600:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787830073.730605:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787830073.730617:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 3025 00000100:00080000:2.0:1787830073.730651:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787830073.730659:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:2.0:1787830073.730701:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8e580 x1874672572657152/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787830073.730725:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787830073.730729:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787830073.730731:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787830073.730735:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 3025 00000100:00080000:2.0:1787830073.730738:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787830073.730742:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:2.0:1787830073.730754:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787830073.730767:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8d680 x1874672572657280/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787830073.730777:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787830073.730780:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1787830073.730782:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787830073.730786:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 3025 00000100:00080000:2.0:1787830073.730789:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787830073.730793:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:2.0:1787830073.730798:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787830073.730809:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8f480 x1874672572657408/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787830073.730965:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8e580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572657152:192.168.203.129@tcp:250:kworker.0 00000800:00000100:0.0:1787830073.731039:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787830073.731083:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830073/real 1787830073] req@ffff8cb998a8e580 x1874672572657152/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830098 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830073.731102:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8e580 x1874672572657152/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830098 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830073.731113:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830073.731117:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787830073.731121:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8e580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572657152:192.168.203.129@tcp:250:kworker.0 00000100:00100000:0.0:1787830073.731136:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8d680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572657280:192.168.203.129@tcp:38:kworker.0 00000800:00000100:0.0:1787830073.731231:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787830073.731245:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830073/real 1787830073] req@ffff8cb998a8d680 x1874672572657280/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830098 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830073.731324:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8d680 x1874672572657280/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830098 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830073.731333:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830073.731337:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787830073.731340:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8d680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572657280:192.168.203.129@tcp:38:kworker.0 00000100:00100000:0.0:1787830073.731347:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572657408:192.168.203.129@tcp:38:kworker.0 00000800:00000100:0.0:1787830073.731370:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787830073.731381:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830073/real 1787830073] req@ffff8cb998a8f480 x1874672572657408/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830098 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830073.731392:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8f480 x1874672572657408/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830098 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830073.731401:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830073.731404:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787830073.731407:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572657408:192.168.203.129@tcp:38:kworker.0 00010000:00010000:0.0:1787830074.564827:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787830075.586599:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787830076.610475:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787830077.635765:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787830078.662004:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787830078.851399:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787830078.851436:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787830078.851441:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787830078.851453:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 3031 00000100:00080000:2.0:1787830078.851467:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787830078.851475:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:2.0:1787830078.851518:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8c000 x1874672572657536/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787830078.851550:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787830078.851554:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787830078.851556:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787830078.851560:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 3031 00000100:00080000:2.0:1787830078.851563:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787830078.851569:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:2.0:1787830078.851582:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787830078.851596:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8cb40 x1874672572657664/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787830078.851608:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787830078.851613:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1787830078.851615:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787830078.851622:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 3031 00000100:00100000:1.0:1787830078.851626:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572657536:192.168.203.129@tcp:250:kworker.0 00000100:00080000:2.0:1787830078.851627:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787830078.851633:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:2.0:1787830078.851645:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787830078.851661:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8ed00 x1874672572657792/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1787830078.851715:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787830078.851763:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830078/real 1787830078] req@ffff8cb998a8c000 x1874672572657536/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830103 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830078.851783:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8c000 x1874672572657536/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830103 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830078.851794:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830078.851811:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787830078.851815:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572657536:192.168.203.129@tcp:250:kworker.0 00000100:00100000:1.0:1787830078.851840:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8cb40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572657664:192.168.203.129@tcp:38:kworker.0 00000800:00000100:1.0:1787830078.851876:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787830078.851889:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830078/real 1787830078] req@ffff8cb998a8cb40 x1874672572657664/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830103 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830078.851900:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8cb40 x1874672572657664/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830103 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830078.851909:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830078.851913:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787830078.851917:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8cb40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572657664:192.168.203.129@tcp:38:kworker.0 00000100:00100000:1.0:1787830078.851930:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8ed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572657792:192.168.203.129@tcp:38:kworker.0 00000800:00000100:1.0:1787830078.851953:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787830078.851964:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830078/real 1787830078] req@ffff8cb998a8ed00 x1874672572657792/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830103 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830078.851975:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8ed00 x1874672572657792/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830103 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830078.851982:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830078.851985:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787830078.851988:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8ed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572657792:192.168.203.129@tcp:38:kworker.0 00010000:00010000:0.0:1787830079.685257:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787830080.706579:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787830081.732539:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787830082.754496:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787830083.778903:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787830083.974525:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787830083.974562:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787830083.974566:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787830083.974576:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 3036 00000100:00080000:3.0:1787830083.974587:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787830083.974594:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:3.0:1787830083.974624:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb992314780 x1874672572657920/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787830083.974656:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787830083.974660:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787830083.974662:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787830083.974666:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 3036 00000100:00080000:3.0:1787830083.974669:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787830083.974673:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:3.0:1787830083.974682:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787830083.974692:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb992314b40 x1874672572658048/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787830083.974701:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787830083.974704:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1787830083.974706:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787830083.974709:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 3036 00000100:00080000:3.0:1787830083.974712:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787830083.974716:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:3.0:1787830083.974720:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787830083.974730:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb992317480 x1874672572658176/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787830083.974743:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb992314780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572657920:192.168.203.129@tcp:250:kworker.0 00000800:00000100:1.0:1787830083.974823:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787830083.974863:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830083/real 1787830083] req@ffff8cb992314780 x1874672572657920/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830108 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830083.974882:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb992314780 x1874672572657920/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830108 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830083.974892:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830083.974896:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787830083.974900:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb992314780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572657920:192.168.203.129@tcp:250:kworker.0 00000100:00100000:1.0:1787830083.974907:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb992314b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572658048:192.168.203.129@tcp:38:kworker.0 00000800:00000100:1.0:1787830083.974930:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787830083.974942:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830083/real 1787830083] req@ffff8cb992314b40 x1874672572658048/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830108 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830083.974954:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb992314b40 x1874672572658048/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830108 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830083.974961:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830083.974964:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787830083.974967:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb992314b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572658048:192.168.203.129@tcp:38:kworker.0 00000100:00100000:1.0:1787830083.974991:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb992317480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572658176:192.168.203.129@tcp:38:kworker.0 00000800:00000100:1.0:1787830083.975011:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787830083.975023:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830083/real 1787830083] req@ffff8cb992317480 x1874672572658176/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830108 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830083.975048:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb992317480 x1874672572658176/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830108 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830083.975056:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830083.975076:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787830083.975080:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb992317480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572658176:192.168.203.129@tcp:38:kworker.0 00010000:00010000:0.0:1787830084.802539:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787830085.826559:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787830086.850464:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787830087.874402:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787830088.898431:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787830089.090365:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787830089.090377:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787830089.090381:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787830089.090391:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 3041 00000100:00080000:3.0:1787830089.090402:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787830089.090410:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:3.0:1787830089.090445:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb992316580 x1874672572658304/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787830089.090483:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787830089.090487:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787830089.090489:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787830089.090493:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 3041 00000100:00080000:3.0:1787830089.090497:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787830089.090501:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:3.0:1787830089.090511:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787830089.090525:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb992317480 x1874672572658432/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787830089.090534:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787830089.090537:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1787830089.090539:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787830089.090543:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 3041 00000100:00080000:3.0:1787830089.090546:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787830089.090549:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:3.0:1787830089.090568:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787830089.090580:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb992314b40 x1874672572658560/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787830089.090657:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb992316580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572658304:192.168.203.129@tcp:250:kworker.0 00000800:00000100:1.0:1787830089.090724:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787830089.090771:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830089/real 1787830089] req@ffff8cb992316580 x1874672572658304/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830114 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830089.090789:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb992316580 x1874672572658304/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830114 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830089.090799:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830089.090803:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787830089.090807:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb992316580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572658304:192.168.203.129@tcp:250:kworker.0 00000100:00100000:1.0:1787830089.090814:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb992317480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572658432:192.168.203.129@tcp:38:kworker.0 00000800:00000100:1.0:1787830089.090835:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787830089.090848:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830089/real 1787830089] req@ffff8cb992317480 x1874672572658432/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830114 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830089.090859:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb992317480 x1874672572658432/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830114 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830089.090880:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830089.090883:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787830089.090887:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb992317480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572658432:192.168.203.129@tcp:38:kworker.0 00000100:00100000:1.0:1787830089.090893:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb992314b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572658560:192.168.203.129@tcp:38:kworker.0 00000800:00000100:1.0:1787830089.090915:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787830089.090926:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830089/real 1787830089] req@ffff8cb992314b40 x1874672572658560/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830114 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830089.090937:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb992314b40 x1874672572658560/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830114 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830089.090944:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830089.090960:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787830089.090963:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb992314b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572658560:192.168.203.129@tcp:38:kworker.0 00010000:00010000:0.0:1787830089.922454:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787830090.946865:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787830091.970472:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787830092.997104:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787830094.019391:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787830094.210476:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787830094.210486:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787830094.210489:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787830094.210498:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 3046 00000100:00080000:2.0:1787830094.210509:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787830094.210516:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:2.0:1787830094.210552:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8f840 x1874672572658688/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787830094.210583:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787830094.210586:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787830094.210588:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787830094.210592:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 3046 00000100:00080000:2.0:1787830094.210595:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787830094.210599:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:2.0:1787830094.210608:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787830094.210619:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8e940 x1874672572658816/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787830094.210629:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787830094.210632:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1787830094.210634:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787830094.210637:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 3046 00000100:00080000:2.0:1787830094.210640:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787830094.210642:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:2.0:1787830094.210659:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787830094.210670:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8ed00 x1874672572658944/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787830094.210735:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8f840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572658688:192.168.203.129@tcp:250:kworker.0 00000800:00000100:1.0:1787830094.210789:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787830094.210824:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830094/real 1787830094] req@ffff8cb998a8f840 x1874672572658688/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830119 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830094.210840:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8f840 x1874672572658688/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830119 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830094.210851:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830094.210854:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787830094.210864:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8f840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572658688:192.168.203.129@tcp:250:kworker.0 00000100:00100000:1.0:1787830094.210871:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8e940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572658816:192.168.203.129@tcp:38:kworker.0 00000800:00000100:1.0:1787830094.210892:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787830094.210903:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830094/real 1787830094] req@ffff8cb998a8e940 x1874672572658816/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830119 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830094.210914:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8e940 x1874672572658816/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830119 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830094.210922:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830094.210925:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787830094.210928:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8e940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572658816:192.168.203.129@tcp:38:kworker.0 00000100:00100000:1.0:1787830094.210934:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8ed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572658944:192.168.203.129@tcp:38:kworker.0 00000800:00000100:1.0:1787830094.210948:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787830094.210957:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830094/real 1787830094] req@ffff8cb998a8ed00 x1874672572658944/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830119 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830094.210968:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8ed00 x1874672572658944/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830119 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830094.210976:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830094.210979:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787830094.210983:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8ed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572658944:192.168.203.129@tcp:38:kworker.0 00010000:00010000:0.0:1787830095.043185:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787830096.066459:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787830097.090849:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787830098.114561:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787830099.139043:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787830099.332998:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787830099.333008:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787830099.333012:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787830099.333021:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 3051 00000100:00080000:3.0:1787830099.333031:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787830099.333048:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:3.0:1787830099.333082:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb992314780 x1874672572659072/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787830099.333117:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787830099.333121:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787830099.333123:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787830099.333127:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 3051 00000100:00080000:3.0:1787830099.333131:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787830099.333136:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:3.0:1787830099.333145:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787830099.333157:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb992314f00 x1874672572659200/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787830099.333167:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787830099.333258:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1787830099.333275:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787830099.333280:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 3051 00000100:00080000:3.0:1787830099.333297:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787830099.333302:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:3.0:1787830099.333307:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787830099.333326:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb992315e00 x1874672572659328/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787830099.334712:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb992314780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572659072:192.168.203.129@tcp:250:kworker.0 00000800:00000100:1.0:1787830099.334778:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787830099.334841:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830099/real 1787830099] req@ffff8cb992314780 x1874672572659072/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830124 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830099.334860:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb992314780 x1874672572659072/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830124 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830099.334871:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830099.334874:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787830099.334877:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb992314780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572659072:192.168.203.129@tcp:250:kworker.0 00000100:00100000:1.0:1787830099.334882:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb992314f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572659200:192.168.203.129@tcp:38:kworker.0 00000800:00000100:1.0:1787830099.334914:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787830099.334927:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830099/real 1787830099] req@ffff8cb992314f00 x1874672572659200/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830124 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830099.334938:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb992314f00 x1874672572659200/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830124 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830099.334946:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830099.334949:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787830099.334952:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb992314f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572659200:192.168.203.129@tcp:38:kworker.0 00000100:00100000:1.0:1787830099.334958:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb992315e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572659328:192.168.203.129@tcp:38:kworker.0 00000800:00000100:1.0:1787830099.334991:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787830099.335001:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830099/real 1787830099] req@ffff8cb992315e00 x1874672572659328/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830124 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830099.335012:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb992315e00 x1874672572659328/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830124 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830099.335030:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830099.335033:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787830099.335036:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb992315e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572659328:192.168.203.129@tcp:38:kworker.0 00010000:00010000:0.0:1787830100.162545:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787830101.186461:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787830102.210534:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787830103.235476:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787830104.259867:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787830104.450436:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787830104.450444:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787830104.450449:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787830104.450459:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 3056 00000100:00080000:3.0:1787830104.450469:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787830104.450476:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:3.0:1787830104.450508:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb992316940 x1874672572659456/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787830104.450537:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787830104.450541:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787830104.450543:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787830104.450546:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 3056 00000100:00080000:3.0:1787830104.450550:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787830104.450555:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:3.0:1787830104.450564:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787830104.450575:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb992315a40 x1874672572659584/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787830104.450586:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787830104.450602:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1787830104.450605:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787830104.450609:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 3056 00000100:00080000:3.0:1787830104.450612:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787830104.450616:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:3.0:1787830104.450622:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787830104.450633:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb9923170c0 x1874672572659712/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787830104.450755:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb992316940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572659456:192.168.203.129@tcp:250:kworker.0 00000800:00000100:0.0:1787830104.450816:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787830104.450857:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830104/real 1787830104] req@ffff8cb992316940 x1874672572659456/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830129 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830104.450888:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb992316940 x1874672572659456/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830129 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830104.452370:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830104.452377:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787830104.452382:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb992316940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572659456:192.168.203.129@tcp:250:kworker.0 00000100:00100000:0.0:1787830104.452393:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb992315a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572659584:192.168.203.129@tcp:38:kworker.0 00000800:00000100:0.0:1787830104.452439:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787830104.452461:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830104/real 1787830104] req@ffff8cb992315a40 x1874672572659584/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830129 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830104.452474:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb992315a40 x1874672572659584/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830129 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830104.452482:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830104.452485:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787830104.452488:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb992315a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572659584:192.168.203.129@tcp:38:kworker.0 00000100:00100000:0.0:1787830104.452494:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb9923170c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572659712:192.168.203.129@tcp:38:kworker.0 00000800:00000100:0.0:1787830104.452512:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787830104.452523:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830104/real 1787830104] req@ffff8cb9923170c0 x1874672572659712/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830129 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830104.452534:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb9923170c0 x1874672572659712/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830129 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830104.452555:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830104.452559:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787830104.452562:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb9923170c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572659712:192.168.203.129@tcp:38:kworker.0 00010000:00010000:0.0:1787830105.282476:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787830106.308209:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787830107.330924:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787830108.354634:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787830109.378452:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787830109.570618:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787830109.570629:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787830109.570633:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787830109.570641:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 3061 00000100:00080000:0.0:1787830109.570652:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787830109.570659:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:0.0:1787830109.570693:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c69a40 x1874672572659840/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787830109.570726:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787830109.570730:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787830109.570732:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787830109.570736:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 3061 00000100:00080000:0.0:1787830109.570739:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787830109.570743:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:0.0:1787830109.570753:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787830109.570766:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c6b840 x1874672572659968/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787830109.570791:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787830109.570795:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1787830109.570797:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1787830109.570800:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c69a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572659840:192.168.203.129@tcp:250:kworker.0 00000100:00080000:0.0:1787830109.570804:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 3061 00000100:00080000:0.0:1787830109.570812:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787830109.570816:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:0.0:1787830109.570822:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787830109.570835:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c68f00 x1874672572660096/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787830109.570958:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c6b840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572659968:192.168.203.129@tcp:38:kworker.0 00000100:00100000:1.0:1787830109.570993:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c68f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572660096:192.168.203.129@tcp:38:kworker.0 00010000:00010000:0.0:1787830110.402558:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787830111.426785:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787830112.450433:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:2.0:1787830112.642669:0:2382:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.129:988 00000400:00000100:2.0:1787830112.642719:0:2382:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.129@tcp at host 192.168.203.129 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:2.0:1787830112.642736:0:2382:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.129@tcp 00000800:00000100:2.0:1787830112.642780:0:2382:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.129@tcp 00000800:00000100:2.0:1787830112.642789:0:2382:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.129@tcp 00000100:00100000:1.0:1787830112.642928:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830109/real 1787830112] req@ffff8cb999c6b840 x1874672572659968/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830134 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830112.642951:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c6b840 x1874672572659968/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830134 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830112.642963:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830112.642967:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787830112.642972:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c6b840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572659968:192.168.203.129@tcp:38:kworker.0 00000100:00100000:1.0:1787830112.642979:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830109/real 1787830112] req@ffff8cb999c68f00 x1874672572660096/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830134 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830112.642991:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c68f00 x1874672572660096/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830134 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830112.642998:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830112.643001:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787830112.643016:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c68f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572660096:192.168.203.129@tcp:38:kworker.0 00000100:00100000:1.0:1787830112.643023:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830109/real 1787830112] req@ffff8cb999c69a40 x1874672572659840/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830134 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830112.643035:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c69a40 x1874672572659840/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830134 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830112.643043:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830112.643046:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787830112.643049:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c69a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572659840:192.168.203.129@tcp:250:kworker.0 00010000:00010000:0.0:1787830113.474550:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787830114.498677:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787830114.690427:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787830114.690438:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787830114.690442:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787830114.690452:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 3066 00000100:00080000:2.0:1787830114.690463:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787830114.690471:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:2.0:1787830114.690501:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8cb40 x1874672572660224/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787830114.690527:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787830114.690531:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787830114.690533:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787830114.690537:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 3066 00000100:00080000:2.0:1787830114.690540:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787830114.690544:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:2.0:1787830114.690553:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787830114.690564:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8c000 x1874672572660352/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787830114.690589:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787830114.690600:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00100000:1.0:1787830114.690602:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8cb40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572660224:192.168.203.129@tcp:250:kworker.0 00000100:00080000:2.0:1787830114.690604:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787830114.690612:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 3066 00000100:00080000:2.0:1787830114.690616:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787830114.690621:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:2.0:1787830114.690627:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787830114.690641:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8f480 x1874672572660480/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1787830114.690655:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787830114.690684:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830114/real 1787830114] req@ffff8cb998a8cb40 x1874672572660224/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830139 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830114.690696:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8cb40 x1874672572660224/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830139 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830114.690702:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830114.690705:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787830114.690708:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8cb40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572660224:192.168.203.129@tcp:250:kworker.0 00000100:00100000:1.0:1787830114.690726:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572660352:192.168.203.129@tcp:38:kworker.0 00000800:00000100:1.0:1787830114.690739:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787830114.690746:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830114/real 1787830114] req@ffff8cb998a8c000 x1874672572660352/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830139 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830114.690763:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8c000 x1874672572660352/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830139 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830114.690769:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830114.690771:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787830114.690773:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572660352:192.168.203.129@tcp:38:kworker.0 00000100:00100000:1.0:1787830114.690776:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572660480:192.168.203.129@tcp:38:kworker.0 00000800:00000100:1.0:1787830114.690787:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787830114.690793:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830114/real 1787830114] req@ffff8cb998a8f480 x1874672572660480/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830139 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830114.690800:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8f480 x1874672572660480/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830139 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830114.690804:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830114.690806:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787830114.690808:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572660480:192.168.203.129@tcp:38:kworker.0 00010000:00010000:0.0:1787830115.522435:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787830116.546428:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787830117.572073:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787830118.594572:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787830119.622618:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787830119.810417:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787830119.810427:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787830119.810431:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787830119.810441:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 3071 00000100:00080000:0.0:1787830119.810452:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787830119.810459:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:0.0:1787830119.810497:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c6bc00 x1874672572660608/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787830119.810541:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787830119.810546:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787830119.810548:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787830119.810552:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 3071 00000100:00080000:0.0:1787830119.810557:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787830119.810562:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:0.0:1787830119.810572:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787830119.810586:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c6b480 x1874672572660736/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787830119.810595:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787830119.810598:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1787830119.810600:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787830119.810603:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 3071 00000100:00080000:0.0:1787830119.810606:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787830119.810610:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:0.0:1787830119.810614:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787830119.810624:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c69e00 x1874672572660864/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787830119.810702:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c6bc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572660608:192.168.203.129@tcp:250:kworker.0 00000800:00000100:1.0:1787830119.810764:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787830119.810802:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830119/real 1787830119] req@ffff8cb999c6bc00 x1874672572660608/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830144 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830119.810830:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c6bc00 x1874672572660608/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830144 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830119.810852:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830119.810856:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787830119.810860:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c6bc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572660608:192.168.203.129@tcp:250:kworker.0 00000100:00100000:1.0:1787830119.810867:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c6b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572660736:192.168.203.129@tcp:38:kworker.0 00000800:00000100:1.0:1787830119.810891:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787830119.810904:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830119/real 1787830119] req@ffff8cb999c6b480 x1874672572660736/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830144 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830119.810915:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c6b480 x1874672572660736/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830144 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830119.810923:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830119.810926:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787830119.810929:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c6b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572660736:192.168.203.129@tcp:38:kworker.0 00000100:00100000:1.0:1787830119.810934:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c69e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572660864:192.168.203.129@tcp:38:kworker.0 00000800:00000100:1.0:1787830119.810953:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787830119.810963:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830119/real 1787830119] req@ffff8cb999c69e00 x1874672572660864/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830144 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830119.810974:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c69e00 x1874672572660864/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830144 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830119.810993:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830119.810997:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787830119.811000:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c69e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572660864:192.168.203.129@tcp:38:kworker.0 00010000:00010000:0.0:1787830120.642480:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787830121.666467:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787830122.690731:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787830123.714454:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787830124.738442:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787830124.930509:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787830124.930519:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787830124.930523:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787830124.930533:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 3077 00000100:00080000:2.0:1787830124.930543:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787830124.930551:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:2.0:1787830124.930585:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8d680 x1874672572660992/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787830124.930615:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787830124.930619:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787830124.930621:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787830124.930624:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 3077 00000100:00080000:2.0:1787830124.930645:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787830124.930649:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:2.0:1787830124.930659:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787830124.930672:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8e580 x1874672572661120/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787830124.930675:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8d680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572660992:192.168.203.129@tcp:250:kworker.0 00000100:00080000:2.0:1787830124.930690:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787830124.930693:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1787830124.930695:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787830124.930699:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 3077 00000100:00080000:2.0:1787830124.930702:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787830124.930706:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:2.0:1787830124.930710:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787830124.930721:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8cf00 x1874672572661248/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1787830124.931702:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787830124.931750:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830124/real 1787830124] req@ffff8cb998a8d680 x1874672572660992/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830149 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830124.931769:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8d680 x1874672572660992/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830149 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830124.931779:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830124.931795:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787830124.931799:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8d680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572660992:192.168.203.129@tcp:250:kworker.0 00000100:00100000:1.0:1787830124.931822:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8e580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572661120:192.168.203.129@tcp:38:kworker.0 00000800:00000100:1.0:1787830124.931850:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787830124.931862:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830124/real 1787830124] req@ffff8cb998a8e580 x1874672572661120/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830149 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830124.931873:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8e580 x1874672572661120/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830149 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830124.931881:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830124.931884:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787830124.931887:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8e580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572661120:192.168.203.129@tcp:38:kworker.0 00000100:00100000:1.0:1787830124.931893:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8cf00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572661248:192.168.203.129@tcp:38:kworker.0 00000800:00000100:1.0:1787830124.931910:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787830124.931921:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830124/real 1787830124] req@ffff8cb998a8cf00 x1874672572661248/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830149 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830124.931943:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8cf00 x1874672572661248/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830149 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830124.931948:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830124.931950:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787830124.931953:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8cf00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572661248:192.168.203.129@tcp:38:kworker.0 00010000:00010000:2.0:1787830125.762565:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787830126.786478:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787830127.810518:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787830128.835810:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787830129.861547:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787830130.051091:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787830130.051102:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787830130.051106:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787830130.051115:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 3082 00000100:00080000:2.0:1787830130.051127:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787830130.051135:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:2.0:1787830130.051169:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8c780 x1874672572661376/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787830130.051222:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787830130.051226:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787830130.051227:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787830130.051231:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 3082 00000100:00080000:2.0:1787830130.051235:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787830130.051240:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:2.0:1787830130.051250:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787830130.051282:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8f0c0 x1874672572661504/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787830130.051293:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787830130.051296:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1787830130.051298:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787830130.051301:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 3082 00000100:00080000:2.0:1787830130.051304:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787830130.051307:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:2.0:1787830130.051312:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787830130.051322:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8fc00 x1874672572661632/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787830130.051513:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8c780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572661376:192.168.203.129@tcp:250:kworker.0 00000800:00000100:1.0:1787830130.051570:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787830130.051611:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830130/real 1787830130] req@ffff8cb998a8c780 x1874672572661376/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830155 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830130.051628:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8c780 x1874672572661376/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830155 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830130.051639:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830130.051643:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787830130.051647:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8c780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572661376:192.168.203.129@tcp:250:kworker.0 00000100:00100000:1.0:1787830130.051654:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8f0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572661504:192.168.203.129@tcp:38:kworker.0 00000800:00000100:1.0:1787830130.051674:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787830130.051686:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830130/real 1787830130] req@ffff8cb998a8f0c0 x1874672572661504/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830155 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830130.051697:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8f0c0 x1874672572661504/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830155 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830130.051717:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830130.051721:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787830130.051725:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8f0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572661504:192.168.203.129@tcp:38:kworker.0 00000100:00100000:1.0:1787830130.051731:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8fc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572661632:192.168.203.129@tcp:38:kworker.0 00000800:00000100:1.0:1787830130.051766:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787830130.051778:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830130/real 1787830130] req@ffff8cb998a8fc00 x1874672572661632/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830155 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830130.051790:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8fc00 x1874672572661632/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830155 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830130.051797:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830130.051800:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787830130.051804:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8fc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572661632:192.168.203.129@tcp:38:kworker.0 00010000:00010000:0.0:1787830130.882561:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787830131.906970:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787830132.930435:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787830133.954490:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787830134.978456:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787830135.170866:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787830135.170876:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787830135.170880:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787830135.170908:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 3087 00000100:00080000:2.0:1787830135.170918:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787830135.170925:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:2.0:1787830135.170956:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8e1c0 x1874672572661760/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787830135.170980:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787830135.170983:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787830135.170985:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787830135.170989:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 3087 00000100:00080000:2.0:1787830135.170992:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787830135.170995:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:2.0:1787830135.171004:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787830135.171016:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8da40 x1874672572661888/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787830135.171026:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8e1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572661760:192.168.203.129@tcp:250:kworker.0 00000100:00080000:2.0:1787830135.171028:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787830135.171035:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1787830135.171037:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787830135.171041:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 3087 00000100:00080000:2.0:1787830135.171044:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787830135.171048:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:2.0:1787830135.171052:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787830135.171064:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8de00 x1874672572662016/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1787830135.171088:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787830135.171129:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830135/real 1787830135] req@ffff8cb998a8e1c0 x1874672572661760/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830160 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830135.171147:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8e1c0 x1874672572661760/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830160 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830135.171155:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830135.171157:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787830135.171165:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8e1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572661760:192.168.203.129@tcp:250:kworker.0 00000100:00100000:1.0:1787830135.171193:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8da40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572661888:192.168.203.129@tcp:38:kworker.0 00000800:00000100:1.0:1787830135.171216:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787830135.171228:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830135/real 1787830135] req@ffff8cb998a8da40 x1874672572661888/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830160 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830135.171237:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8da40 x1874672572661888/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830160 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830135.171242:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830135.171244:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787830135.171246:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8da40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572661888:192.168.203.129@tcp:38:kworker.0 00000100:00100000:1.0:1787830135.171250:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8de00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572662016:192.168.203.129@tcp:38:kworker.0 00000800:00000100:1.0:1787830135.171286:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787830135.171297:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830135/real 1787830135] req@ffff8cb998a8de00 x1874672572662016/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830160 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830135.171308:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8de00 x1874672572662016/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830160 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830135.171317:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830135.171320:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787830135.171323:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8de00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572662016:192.168.203.129@tcp:38:kworker.0 00010000:00010000:0.0:1787830136.002475:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787830137.026439:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787830138.050757:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787830139.074822:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787830140.099093:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787830140.290363:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787830140.290374:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787830140.290379:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787830140.290388:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 3092 00000100:00080000:2.0:1787830140.290399:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787830140.290407:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:2.0:1787830140.290443:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8d2c0 x1874672572662144/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787830140.290520:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8d2c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572662144:192.168.203.129@tcp:250:kworker.0 00000800:00000100:1.0:1787830140.290579:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00080000:2.0:1787830140.290625:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787830140.290630:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787830140.290633:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787830140.290638:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 3092 00000100:00080000:2.0:1787830140.290643:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787830140.290648:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:2.0:1787830140.290657:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787830140.290672:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8c3c0 x1874672572662272/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787830140.290798:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830140/real 1787830140] req@ffff8cb998a8d2c0 x1874672572662144/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830165 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830140.290817:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8d2c0 x1874672572662144/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830165 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787830140.290817:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787830140.290821:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1787830140.290823:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787830140.290827:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787830140.290827:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 3092 00000100:00080000:1.0:1787830140.290831:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00080000:2.0:1787830140.290831:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1787830140.290835:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8d2c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572662144:192.168.203.129@tcp:250:kworker.0 00000100:00080000:2.0:1787830140.290842:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:2.0:1787830140.290848:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787830140.290858:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8c3c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572662272:192.168.203.129@tcp:38:kworker.0 00000100:00100000:2.0:1787830140.290862:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8d2c0 x1874672572662400/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1787830140.290885:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787830140.290906:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830140/real 1787830140] req@ffff8cb998a8c3c0 x1874672572662272/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830165 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830140.290918:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8c3c0 x1874672572662272/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830165 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830140.290926:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830140.290929:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787830140.290932:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8c3c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572662272:192.168.203.129@tcp:38:kworker.0 00000100:00100000:1.0:1787830140.290943:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8d2c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572662400:192.168.203.129@tcp:38:kworker.0 00000800:00000100:1.0:1787830140.290974:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787830140.290991:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830140/real 1787830140] req@ffff8cb998a8d2c0 x1874672572662400/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830165 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830140.291003:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8d2c0 x1874672572662400/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830165 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830140.291022:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830140.291026:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787830140.291030:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8d2c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572662400:192.168.203.129@tcp:38:kworker.0 00010000:00010000:2.0:1787830141.122451:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787830142.147025:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787830143.173584:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787830144.194678:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787830145.218656:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787830145.410547:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787830145.410555:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787830145.410558:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787830145.410565:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 3097 00000100:00080000:0.0:1787830145.410574:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787830145.410580:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:0.0:1787830145.410613:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c692c0 x1874672572662528/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787830145.410659:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787830145.410663:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787830145.410665:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787830145.410669:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 3097 00000100:00080000:0.0:1787830145.410673:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787830145.410676:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:0.0:1787830145.410686:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787830145.410703:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c692c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572662528:192.168.203.129@tcp:250:kworker.0 00000100:00100000:0.0:1787830145.410716:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c69680 x1874672572662656/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787830145.410729:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787830145.410732:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1787830145.410735:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787830145.410740:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 3097 00000100:00080000:0.0:1787830145.410758:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787830145.410762:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:0.0:1787830145.410768:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000800:00000100:1.0:1787830145.410769:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787830145.410783:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c6a1c0 x1874672572662784/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787830145.410815:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830145/real 1787830145] req@ffff8cb999c692c0 x1874672572662528/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830170 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830145.410833:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c692c0 x1874672572662528/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830170 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830145.410843:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830145.410847:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787830145.410850:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c692c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572662528:192.168.203.129@tcp:250:kworker.0 00000100:00100000:1.0:1787830145.410873:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c69680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572662656:192.168.203.129@tcp:38:kworker.0 00000800:00000100:1.0:1787830145.410895:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787830145.410907:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830145/real 1787830145] req@ffff8cb999c69680 x1874672572662656/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830170 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830145.410919:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c69680 x1874672572662656/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830170 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830145.410927:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830145.410930:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787830145.410933:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c69680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572662656:192.168.203.129@tcp:38:kworker.0 00000100:00100000:1.0:1787830145.410947:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c6a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572662784:192.168.203.129@tcp:38:kworker.0 00000800:00000100:1.0:1787830145.410966:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787830145.410977:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830145/real 1787830145] req@ffff8cb999c6a1c0 x1874672572662784/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830170 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830145.410988:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c6a1c0 x1874672572662784/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830170 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830145.410995:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830145.410998:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787830145.411000:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c6a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572662784:192.168.203.129@tcp:38:kworker.0 00010000:00010000:0.0:1787830146.242452:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787830147.266543:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787830148.290954:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787830149.314585:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787830150.338544:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787830150.530412:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787830150.530423:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787830150.530427:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787830150.530437:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 3102 00000100:00080000:1.0:1787830150.530448:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787830150.530455:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:1.0:1787830150.530490:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f383c0 x1874672572662912/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787830150.530521:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787830150.530526:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787830150.530528:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787830150.530533:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 3102 00000100:00080000:1.0:1787830150.530537:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787830150.530556:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:0.0:1787830150.530563:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f383c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572662912:192.168.203.129@tcp:250:kworker.0 00000008:00100000:1.0:1787830150.530565:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787830150.530584:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f3a580 x1874672572663040/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787830150.530596:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787830150.530600:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1787830150.530602:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787830150.530615:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 3102 00000100:00080000:1.0:1787830150.530631:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787830150.530636:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000800:00000100:0.0:1787830150.530642:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000008:00100000:1.0:1787830150.530642:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787830150.530658:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f392c0 x1874672572663168/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787830150.530678:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830150/real 1787830150] req@ffff8cb999f383c0 x1874672572662912/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830175 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830150.530692:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f383c0 x1874672572662912/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830175 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830150.530699:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830150.530702:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787830150.530704:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f383c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572662912:192.168.203.129@tcp:250:kworker.0 00000100:00100000:0.0:1787830150.530734:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f3a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572663040:192.168.203.129@tcp:38:kworker.0 00000800:00000100:0.0:1787830150.530756:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787830150.530767:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830150/real 1787830150] req@ffff8cb999f3a580 x1874672572663040/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830175 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830150.530779:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f3a580 x1874672572663040/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830175 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830150.530787:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830150.530790:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787830150.530794:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f3a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572663040:192.168.203.129@tcp:38:kworker.0 00000100:00100000:0.0:1787830150.530799:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f392c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572663168:192.168.203.129@tcp:38:kworker.0 00000800:00000100:0.0:1787830150.530816:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787830150.530826:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830150/real 1787830150] req@ffff8cb999f392c0 x1874672572663168/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830175 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830150.530837:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f392c0 x1874672572663168/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830175 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830150.530846:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830150.530849:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787830150.530852:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f392c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572663168:192.168.203.129@tcp:38:kworker.0 00010000:00010000:0.0:1787830151.362835:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787830152.386515:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787830153.410707:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787830154.434453:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787830155.458542:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787830155.650752:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787830155.650762:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787830155.650767:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787830155.650777:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 3107 00000100:00080000:0.0:1787830155.650787:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787830155.650795:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:0.0:1787830155.650831:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c68000 x1874672572663296/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787830155.651070:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c68000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572663296:192.168.203.129@tcp:250:kworker.0 00000800:00000100:1.0:1787830155.651131:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787830155.651561:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830155/real 1787830155] req@ffff8cb999c68000 x1874672572663296/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830180 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830155.651590:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c68000 x1874672572663296/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830180 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830155.651601:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830155.651605:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787830155.651608:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c68000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572663296:192.168.203.129@tcp:250:kworker.0 00000100:00080000:0.0:1787830155.651693:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787830155.651698:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787830155.651700:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787830155.651705:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 3107 00000100:00080000:0.0:1787830155.651710:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787830155.651714:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:0.0:1787830155.651723:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787830155.651739:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c6a580 x1874672572663424/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787830155.651758:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787830155.651762:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1787830155.651764:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787830155.651768:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 3107 00000100:00080000:0.0:1787830155.651771:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787830155.651775:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:0.0:1787830155.651780:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787830155.651799:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c68000 x1874672572663552/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787830155.651963:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c6a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572663424:192.168.203.129@tcp:38:kworker.0 00000800:00000100:1.0:1787830155.651996:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787830155.652023:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830155/real 1787830155] req@ffff8cb999c6a580 x1874672572663424/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830180 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830155.652036:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c6a580 x1874672572663424/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830180 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830155.652044:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830155.652048:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787830155.652052:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c6a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572663424:192.168.203.129@tcp:38:kworker.0 00000100:00100000:1.0:1787830155.652059:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c68000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572663552:192.168.203.129@tcp:38:kworker.0 00000800:00000100:1.0:1787830155.652078:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787830155.652089:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830155/real 1787830155] req@ffff8cb999c68000 x1874672572663552/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830180 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830155.652100:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c68000 x1874672572663552/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830180 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830155.652107:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830155.652111:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787830155.652114:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c68000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572663552:192.168.203.129@tcp:38:kworker.0 00010000:00010000:2.0:1787830156.482443:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787830157.506436:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787830158.530467:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787830159.555831:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787830160.578865:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787830160.770365:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787830160.770377:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787830160.770381:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787830160.770397:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 3112 00000100:00080000:0.0:1787830160.770409:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787830160.770416:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:0.0:1787830160.770450:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c6a1c0 x1874672572663680/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787830160.770520:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c6a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572663680:192.168.203.129@tcp:250:kworker.0 00000800:00000100:1.0:1787830160.770592:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00080000:0.0:1787830160.770625:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787830160.770629:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787830160.770631:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787830160.770635:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 3112 00000100:00080000:0.0:1787830160.770639:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787830160.770643:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:0.0:1787830160.770651:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787830160.770662:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c69680 x1874672572663808/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787830160.772416:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830160/real 1787830160] req@ffff8cb999c6a1c0 x1874672572663680/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830185 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830160.772456:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c6a1c0 x1874672572663680/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830185 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830160.772467:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830160.772471:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787830160.772475:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c6a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572663680:192.168.203.129@tcp:250:kworker.0 00000100:00100000:1.0:1787830160.772501:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c69680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572663808:192.168.203.129@tcp:38:kworker.0 00000800:00000100:1.0:1787830160.772553:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787830160.772568:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830160/real 1787830160] req@ffff8cb999c69680 x1874672572663808/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830185 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830160.772580:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c69680 x1874672572663808/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830185 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830160.772587:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830160.772590:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787830160.772593:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c69680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572663808:192.168.203.129@tcp:38:kworker.0 00000100:00080000:0.0:1787830160.772706:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787830160.772711:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1787830160.772713:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787830160.772723:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 3112 00000100:00080000:0.0:1787830160.772728:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787830160.772733:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:0.0:1787830160.772742:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787830160.772759:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c692c0 x1874672572663936/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787830160.772988:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c692c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572663936:192.168.203.129@tcp:38:kworker.0 00000800:00000100:1.0:1787830160.774796:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787830160.776625:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830160/real 1787830160] req@ffff8cb999c692c0 x1874672572663936/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830185 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830160.776649:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c692c0 x1874672572663936/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830185 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830160.776681:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830160.776687:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787830160.776691:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c692c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572663936:192.168.203.129@tcp:38:kworker.0 00010000:00010000:0.0:1787830161.602425:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787830162.626472:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787830163.650477:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787830164.674455:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787830165.698426:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787830165.890378:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787830165.890389:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787830165.890393:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787830165.890404:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 3118 00000100:00080000:0.0:1787830165.890414:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787830165.890419:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:0.0:1787830165.890451:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c69e00 x1874672572664064/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787830165.890475:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787830165.890479:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787830165.890489:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787830165.890493:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 3118 00000100:00080000:0.0:1787830165.890507:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787830165.890511:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:0.0:1787830165.890520:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787830165.890533:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c6b480 x1874672572664192/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787830165.890559:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787830165.890562:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1787830165.890564:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787830165.890567:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 3118 00000100:00080000:0.0:1787830165.890571:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787830165.890575:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:0.0:1787830165.890580:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787830165.890591:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c6bc00 x1874672572664320/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787830165.891333:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c69e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572664064:192.168.203.129@tcp:250:kworker.0 00000800:00000100:1.0:1787830165.891396:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787830165.891437:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830165/real 1787830165] req@ffff8cb999c69e00 x1874672572664064/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830190 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830165.891456:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c69e00 x1874672572664064/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830190 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830165.891467:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830165.891471:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787830165.891475:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c69e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572664064:192.168.203.129@tcp:250:kworker.0 00000100:00100000:1.0:1787830165.891482:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c6b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572664192:192.168.203.129@tcp:38:kworker.0 00000800:00000100:1.0:1787830165.891519:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787830165.891532:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830165/real 1787830165] req@ffff8cb999c6b480 x1874672572664192/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830190 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830165.891545:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c6b480 x1874672572664192/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830190 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830165.891554:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830165.891557:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787830165.891561:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c6b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572664192:192.168.203.129@tcp:38:kworker.0 00000100:00100000:1.0:1787830165.891575:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c6bc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572664320:192.168.203.129@tcp:38:kworker.0 00000800:00000100:1.0:1787830165.891616:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787830165.891629:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830165/real 1787830165] req@ffff8cb999c6bc00 x1874672572664320/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830190 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830165.891641:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c6bc00 x1874672572664320/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830190 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830165.891649:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830165.891652:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787830165.891655:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c6bc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572664320:192.168.203.129@tcp:38:kworker.0 00010000:00010000:2.0:1787830166.722674:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787830167.747039:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787830168.772783:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787830169.794684:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787830170.818509:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787830171.011051:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787830171.011062:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787830171.011066:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787830171.011075:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 3123 00000100:00080000:2.0:1787830171.011086:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787830171.011093:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:2.0:1787830171.011130:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8de00 x1874672572664448/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787830171.011216:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8de00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572664448:192.168.203.129@tcp:250:kworker.0 00000800:00000100:1.0:1787830171.011326:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00080000:2.0:1787830171.012139:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787830171.012160:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787830171.012163:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787830171.012168:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 3123 00000100:00080000:2.0:1787830171.012172:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787830171.012176:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:2.0:1787830171.012184:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787830171.012198:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8da40 x1874672572664576/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787830171.014145:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787830171.014154:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00100000:1.0:1787830171.014200:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830171/real 1787830171] req@ffff8cb998a8de00 x1874672572664448/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830196 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830171.014226:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8de00 x1874672572664448/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830196 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830171.014237:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830171.014242:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787830171.014246:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8de00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572664448:192.168.203.129@tcp:250:kworker.0 00000100:00100000:1.0:1787830171.014300:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8da40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572664576:192.168.203.129@tcp:38:kworker.0 00000100:00080000:2.0:1787830171.014329:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787830171.014337:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 3123 00000100:00080000:2.0:1787830171.014343:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787830171.014351:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000800:00000100:1.0:1787830171.014353:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000008:00100000:2.0:1787830171.014360:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787830171.014382:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8e1c0 x1874672572664704/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787830171.014387:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830171/real 1787830171] req@ffff8cb998a8da40 x1874672572664576/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830196 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830171.014416:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8da40 x1874672572664576/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830196 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830171.014425:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830171.014429:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787830171.014433:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8da40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572664576:192.168.203.129@tcp:38:kworker.0 00000100:00100000:1.0:1787830171.014448:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8e1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572664704:192.168.203.129@tcp:38:kworker.0 00000800:00000100:1.0:1787830171.014481:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787830171.014493:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830171/real 1787830171] req@ffff8cb998a8e1c0 x1874672572664704/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830196 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830171.014506:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8e1c0 x1874672572664704/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830196 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830171.014513:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830171.014808:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787830171.014812:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8e1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572664704:192.168.203.129@tcp:38:kworker.0 00010000:00010000:2.0:1787830171.842468:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787830172.866554:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787830173.890452:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787830174.915029:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787830175.940677:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787830176.130371:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787830176.130381:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787830176.130385:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787830176.130408:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 3128 00000100:00080000:2.0:1787830176.130419:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787830176.130435:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:2.0:1787830176.130468:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8fc00 x1874672572664832/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787830176.130502:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787830176.130505:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787830176.130507:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787830176.130511:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 3128 00000100:00080000:2.0:1787830176.130515:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787830176.130520:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:2.0:1787830176.130529:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787830176.130536:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8fc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572664832:192.168.203.129@tcp:250:kworker.0 00000100:00100000:2.0:1787830176.130544:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8f0c0 x1874672572664960/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787830176.130555:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787830176.130558:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1787830176.130560:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787830176.130564:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 3128 00000100:00080000:2.0:1787830176.130568:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787830176.130572:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:2.0:1787830176.130578:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787830176.130589:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8c780 x1874672572665088/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787830176.130636:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8f0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572664960:192.168.203.129@tcp:38:kworker.0 00000100:00100000:1.0:1787830176.130666:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8c780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572665088:192.168.203.129@tcp:38:kworker.0 00010000:00010000:0.0:1787830176.962517:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787830177.986745:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787830179.011438:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:3.0:1787830179.207648:0:2383:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.129:988 00000400:00000100:3.0:1787830179.207700:0:2383:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.129@tcp at host 192.168.203.129 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:3.0:1787830179.207716:0:2383:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.129@tcp 00000800:00000100:3.0:1787830179.207758:0:2383:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.129@tcp 00000800:00000100:3.0:1787830179.207772:0:2383:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.129@tcp 00000100:00100000:1.0:1787830179.207848:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830176/real 1787830179] req@ffff8cb998a8f0c0 x1874672572664960/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830201 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830179.207974:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8f0c0 x1874672572664960/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830201 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830179.207985:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830179.207990:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787830179.207994:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8f0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572664960:192.168.203.129@tcp:38:kworker.0 00000100:00100000:1.0:1787830179.208000:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830176/real 1787830179] req@ffff8cb998a8c780 x1874672572665088/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830201 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830179.208017:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8c780 x1874672572665088/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830201 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830179.208024:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830179.208028:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787830179.208032:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8c780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572665088:192.168.203.129@tcp:38:kworker.0 00000100:00100000:1.0:1787830179.208038:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830176/real 1787830179] req@ffff8cb998a8fc00 x1874672572664832/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830201 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830179.208079:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8fc00 x1874672572664832/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830201 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830179.208089:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830179.208093:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787830179.208097:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8fc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572664832:192.168.203.129@tcp:250:kworker.0 00010000:00010000:0.0:1787830180.034427:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787830181.063002:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787830181.252353:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787830181.252363:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787830181.252366:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787830181.252375:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 3133 00000100:00080000:0.0:1787830181.252386:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787830181.252488:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:0.0:1787830181.252523:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c69a40 x1874672572665216/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787830181.252552:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787830181.252556:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787830181.252574:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787830181.252578:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 3133 00000100:00080000:0.0:1787830181.252582:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787830181.252586:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:0.0:1787830181.252595:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787830181.252608:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c68f00 x1874672572665344/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787830181.252616:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787830181.252618:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1787830181.252620:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787830181.252623:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 3133 00000100:00080000:0.0:1787830181.252626:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787830181.252629:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:0.0:1787830181.252634:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787830181.252642:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c6b840 x1874672572665472/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787830181.252806:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c69a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572665216:192.168.203.129@tcp:250:kworker.0 00000800:00000100:1.0:1787830181.252870:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787830181.252907:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830181/real 1787830181] req@ffff8cb999c69a40 x1874672572665216/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830206 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830181.252923:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c69a40 x1874672572665216/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830206 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830181.252934:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830181.252938:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787830181.252942:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c69a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572665216:192.168.203.129@tcp:250:kworker.0 00000100:00100000:1.0:1787830181.252949:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c68f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572665344:192.168.203.129@tcp:38:kworker.0 00000800:00000100:1.0:1787830181.252970:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787830181.252981:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830181/real 1787830181] req@ffff8cb999c68f00 x1874672572665344/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830206 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830181.252993:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c68f00 x1874672572665344/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830206 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830181.253001:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830181.253004:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787830181.253007:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c68f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572665344:192.168.203.129@tcp:38:kworker.0 00000100:00100000:1.0:1787830181.253013:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c6b840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572665472:192.168.203.129@tcp:38:kworker.0 00000800:00000100:1.0:1787830181.253374:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787830181.253389:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830181/real 1787830181] req@ffff8cb999c6b840 x1874672572665472/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830206 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830181.253424:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c6b840 x1874672572665472/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830206 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830181.253499:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830181.253503:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787830181.253506:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c6b840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572665472:192.168.203.129@tcp:38:kworker.0 00010000:00010000:2.0:1787830182.082431:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787830183.109468:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787830184.131807:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787830185.156881:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787830186.178628:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787830186.370377:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787830186.370387:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787830186.370392:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787830186.370401:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 3138 00000100:00080000:0.0:1787830186.370410:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787830186.370418:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:0.0:1787830186.370470:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c6b840 x1874672572665600/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787830186.370498:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787830186.370502:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787830186.370503:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787830186.370506:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 3138 00000100:00080000:0.0:1787830186.370510:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787830186.370514:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:0.0:1787830186.370523:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787830186.370535:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c68f00 x1874672572665728/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787830186.370544:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787830186.370547:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1787830186.370549:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787830186.370551:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 3138 00000100:00080000:0.0:1787830186.370554:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787830186.370557:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:0.0:1787830186.370562:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787830186.370571:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c69a40 x1874672572665856/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787830186.370592:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c6b840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572665600:192.168.203.129@tcp:250:kworker.0 00000800:00000100:1.0:1787830186.370654:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787830186.370751:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830186/real 1787830186] req@ffff8cb999c6b840 x1874672572665600/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830211 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830186.370767:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c6b840 x1874672572665600/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830211 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830186.370778:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830186.370782:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787830186.370785:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c6b840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572665600:192.168.203.129@tcp:250:kworker.0 00000100:00100000:1.0:1787830186.370792:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c68f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572665728:192.168.203.129@tcp:38:kworker.0 00000800:00000100:1.0:1787830186.370816:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787830186.370830:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830186/real 1787830186] req@ffff8cb999c68f00 x1874672572665728/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830211 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830186.370841:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c68f00 x1874672572665728/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830211 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830186.370862:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830186.370865:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787830186.370869:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c68f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572665728:192.168.203.129@tcp:38:kworker.0 00000100:00100000:1.0:1787830186.370875:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c69a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572665856:192.168.203.129@tcp:38:kworker.0 00000800:00000100:1.0:1787830186.370895:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787830186.370907:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830186/real 1787830186] req@ffff8cb999c69a40 x1874672572665856/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830211 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830186.370918:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c69a40 x1874672572665856/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830211 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830186.370925:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830186.370928:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787830186.370932:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c69a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572665856:192.168.203.129@tcp:38:kworker.0 00010000:00010000:2.0:1787830187.206302:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787830188.226483:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787830189.251243:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787830190.274528:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787830191.298554:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787830191.490845:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787830191.490855:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787830191.490859:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787830191.490869:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 3143 00000100:00080000:1.0:1787830191.490879:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787830191.490886:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:1.0:1787830191.490919:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f38f00 x1874672572665984/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787830191.490980:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f38f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572665984:192.168.203.129@tcp:250:kworker.0 00000800:00000100:0.0:1787830191.491037:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00080000:1.0:1787830191.491563:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787830191.491574:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787830191.491576:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787830191.491581:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 3143 00000100:00080000:1.0:1787830191.491596:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787830191.491602:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:1.0:1787830191.491612:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787830191.491704:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830191/real 1787830191] req@ffff8cb999f38f00 x1874672572665984/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830216 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830191.491720:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f38f00 x1874672572665984/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830216 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830191.491729:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830191.491732:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787830191.491735:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f38f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572665984:192.168.203.129@tcp:250:kworker.0 00000100:00100000:1.0:1787830191.491955:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f38b40 x1874672572666112/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787830191.491996:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f38b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572666112:192.168.203.129@tcp:38:kworker.0 00000800:00000100:0.0:1787830191.492026:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00080000:1.0:1787830191.492046:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787830191.492050:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1787830191.492051:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787830191.492055:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 3143 00000100:00080000:1.0:1787830191.492058:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787830191.492163:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:1.0:1787830191.492169:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787830191.492182:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f3b0c0 x1874672572666240/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787830191.492220:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830191/real 1787830191] req@ffff8cb999f38b40 x1874672572666112/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830216 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830191.492232:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f38b40 x1874672572666112/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830216 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830191.492239:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830191.492243:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787830191.492246:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f38b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572666112:192.168.203.129@tcp:38:kworker.0 00000100:00100000:0.0:1787830191.492258:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f3b0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572666240:192.168.203.129@tcp:38:kworker.0 00000800:00000100:0.0:1787830191.492293:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787830191.492304:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830191/real 1787830191] req@ffff8cb999f3b0c0 x1874672572666240/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830216 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830191.492315:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f3b0c0 x1874672572666240/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830216 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830191.492552:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830191.492557:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787830191.492560:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f3b0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572666240:192.168.203.129@tcp:38:kworker.0 00010000:00010000:0.0:1787830192.322822:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787830193.348084:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787830194.370582:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787830195.394628:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787830196.421364:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787830196.611182:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787830196.611193:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787830196.611197:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787830196.611207:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 3148 00000100:00080000:0.0:1787830196.611217:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787830196.611227:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:0.0:1787830196.611298:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c6bc00 x1874672572666368/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787830196.611330:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787830196.611334:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787830196.611335:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787830196.611339:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 3148 00000100:00080000:0.0:1787830196.611342:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787830196.611346:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:0.0:1787830196.611355:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787830196.611367:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c6b480 x1874672572666496/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787830196.611377:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787830196.611395:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1787830196.611397:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787830196.611401:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 3148 00000100:00080000:0.0:1787830196.611405:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787830196.611409:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:0.0:1787830196.611414:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787830196.611426:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c69e00 x1874672572666624/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787830196.611629:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c6bc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572666368:192.168.203.129@tcp:250:kworker.0 00000800:00000100:1.0:1787830196.611689:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787830196.611729:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830196/real 1787830196] req@ffff8cb999c6bc00 x1874672572666368/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830221 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830196.611745:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c6bc00 x1874672572666368/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830221 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830196.611765:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830196.611768:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787830196.611772:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c6bc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572666368:192.168.203.129@tcp:250:kworker.0 00000100:00100000:1.0:1787830196.611789:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c6b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572666496:192.168.203.129@tcp:38:kworker.0 00000800:00000100:1.0:1787830196.611814:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787830196.611850:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830196/real 1787830196] req@ffff8cb999c6b480 x1874672572666496/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830221 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830196.611862:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c6b480 x1874672572666496/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830221 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830196.611870:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830196.611873:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787830196.611876:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c6b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572666496:192.168.203.129@tcp:38:kworker.0 00000100:00100000:1.0:1787830196.611882:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c69e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572666624:192.168.203.129@tcp:38:kworker.0 00000800:00000100:1.0:1787830196.611901:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787830196.611913:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830196/real 1787830196] req@ffff8cb999c69e00 x1874672572666624/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830221 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830196.611924:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c69e00 x1874672572666624/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830221 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830196.611931:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830196.611935:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787830196.611937:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c69e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572666624:192.168.203.129@tcp:38:kworker.0 00010000:00010000:2.0:1787830197.442522:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787830198.466797:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787830199.490466:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787830200.514445:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787830201.538618:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787830201.730373:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787830201.730383:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787830201.730387:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787830201.730397:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 3153 00000100:00080000:0.0:1787830201.730408:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787830201.730415:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:0.0:1787830201.730449:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c692c0 x1874672572666752/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787830201.730477:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787830201.730481:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787830201.730483:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787830201.730486:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 3153 00000100:00080000:0.0:1787830201.730490:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787830201.730494:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:0.0:1787830201.730503:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787830201.730525:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c69680 x1874672572666880/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787830201.730549:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787830201.730553:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1787830201.730554:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787830201.730558:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 3153 00000100:00080000:0.0:1787830201.730562:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787830201.730566:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:0.0:1787830201.730571:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787830201.730582:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c6a1c0 x1874672572667008/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787830201.730773:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c692c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572666752:192.168.203.129@tcp:250:kworker.0 00000800:00000100:1.0:1787830201.730831:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787830201.730870:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830201/real 1787830201] req@ffff8cb999c692c0 x1874672572666752/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830226 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830201.730887:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c692c0 x1874672572666752/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830226 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830201.730896:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830201.730900:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787830201.730903:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c692c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572666752:192.168.203.129@tcp:250:kworker.0 00000100:00100000:1.0:1787830201.730926:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c69680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572666880:192.168.203.129@tcp:38:kworker.0 00000800:00000100:1.0:1787830201.731659:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787830201.731678:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830201/real 1787830201] req@ffff8cb999c69680 x1874672572666880/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830226 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830201.731701:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c69680 x1874672572666880/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830226 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830201.732171:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830201.732176:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787830201.732180:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c69680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572666880:192.168.203.129@tcp:38:kworker.0 00000100:00100000:1.0:1787830201.732188:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c6a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572667008:192.168.203.129@tcp:38:kworker.0 00000800:00000100:1.0:1787830201.732221:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787830201.732235:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830201/real 1787830201] req@ffff8cb999c6a1c0 x1874672572667008/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830226 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830201.732289:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c6a1c0 x1874672572667008/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830226 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830201.732298:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830201.732301:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787830201.732304:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c6a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572667008:192.168.203.129@tcp:38:kworker.0 00010000:00010000:2.0:1787830202.562630:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787830203.587303:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787830204.610435:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787830205.634857:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787830206.658783:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787830206.850392:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787830206.850403:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787830206.850407:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787830206.850418:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 3159 00000100:00080000:2.0:1787830206.850428:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787830206.850437:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:2.0:1787830206.850472:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8cf00 x1874672572667136/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787830206.850519:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787830206.850522:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787830206.850524:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787830206.850528:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 3159 00000100:00080000:2.0:1787830206.850533:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787830206.850537:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:2.0:1787830206.850557:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787830206.850571:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8e580 x1874672572667264/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787830206.850581:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787830206.850585:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1787830206.850588:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1787830206.850589:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8cf00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572667136:192.168.203.129@tcp:250:kworker.0 00000100:00080000:2.0:1787830206.850594:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 3159 00000100:00080000:2.0:1787830206.850600:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787830206.850603:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:2.0:1787830206.850608:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787830206.850620:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8d680 x1874672572667392/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1787830206.850667:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787830206.850708:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830206/real 1787830206] req@ffff8cb998a8cf00 x1874672572667136/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830231 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830206.850722:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8cf00 x1874672572667136/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830231 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830206.850729:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830206.850732:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787830206.850735:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8cf00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572667136:192.168.203.129@tcp:250:kworker.0 00000100:00100000:1.0:1787830206.850753:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8e580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572667264:192.168.203.129@tcp:38:kworker.0 00000800:00000100:1.0:1787830206.850768:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787830206.850776:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830206/real 1787830206] req@ffff8cb998a8e580 x1874672572667264/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830231 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830206.850783:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8e580 x1874672572667264/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830231 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830206.850806:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830206.850809:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787830206.850825:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8e580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572667264:192.168.203.129@tcp:38:kworker.0 00000100:00100000:1.0:1787830206.850832:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8d680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572667392:192.168.203.129@tcp:38:kworker.0 00000800:00000100:1.0:1787830206.850861:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787830206.850873:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830206/real 1787830206] req@ffff8cb998a8d680 x1874672572667392/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830231 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830206.850884:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8d680 x1874672572667392/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830231 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830206.850892:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830206.850895:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787830206.850899:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8d680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572667392:192.168.203.129@tcp:38:kworker.0 00010000:00010000:2.0:1787830207.683737:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787830208.706520:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787830209.731248:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787830210.754423:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787830211.778563:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787830211.970400:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787830211.970427:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787830211.970431:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787830211.970440:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 3164 00000100:00080000:2.0:1787830211.970450:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787830211.970457:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:2.0:1787830211.970487:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8f480 x1874672572667520/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787830211.970518:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787830211.970521:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787830211.970523:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787830211.970527:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 3164 00000100:00080000:2.0:1787830211.970530:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787830211.970534:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:2.0:1787830211.970543:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787830211.970558:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572667520:192.168.203.129@tcp:250:kworker.0 00000100:00100000:2.0:1787830211.970558:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8c000 x1874672572667648/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787830211.970574:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787830211.970577:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1787830211.970579:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787830211.970583:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 3164 00000100:00080000:2.0:1787830211.970586:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787830211.970590:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:2.0:1787830211.970595:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787830211.970607:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8cb40 x1874672572667776/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1787830211.970635:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787830211.970676:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830211/real 1787830211] req@ffff8cb998a8f480 x1874672572667520/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830236 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830211.970693:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8f480 x1874672572667520/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830236 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830211.970702:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830211.970706:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787830211.970716:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572667520:192.168.203.129@tcp:250:kworker.0 00000100:00100000:1.0:1787830211.970739:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572667648:192.168.203.129@tcp:38:kworker.0 00000800:00000100:1.0:1787830211.970763:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787830211.970775:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830211/real 1787830211] req@ffff8cb998a8c000 x1874672572667648/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830236 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830211.970787:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8c000 x1874672572667648/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830236 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830211.970794:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830211.970797:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787830211.970800:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572667648:192.168.203.129@tcp:38:kworker.0 00000100:00100000:1.0:1787830211.970806:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8cb40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572667776:192.168.203.129@tcp:38:kworker.0 00000800:00000100:1.0:1787830211.970823:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787830211.970833:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830211/real 1787830211] req@ffff8cb998a8cb40 x1874672572667776/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830236 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830211.970844:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8cb40 x1874672572667776/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830236 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830211.970851:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830211.970854:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787830211.970857:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8cb40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572667776:192.168.203.129@tcp:38:kworker.0 00010000:00010000:1.0:1787830212.802931:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787830213.826463:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787830214.850858:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787830215.874520:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787830216.898957:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787830217.090742:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787830217.090756:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787830217.090761:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787830217.090773:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 3169 00000100:00080000:1.0:1787830217.090786:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787830217.090794:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:1.0:1787830217.090884:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f3b480 x1874672572667904/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787830217.090969:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787830217.090973:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787830217.090975:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787830217.090979:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 3169 00000100:00080000:1.0:1787830217.091037:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787830217.091043:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:1.0:1787830217.091055:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787830217.091069:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f3b840 x1874672572668032/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787830217.091079:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787830217.091082:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1787830217.091084:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787830217.091088:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 3169 00000100:00080000:1.0:1787830217.091091:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787830217.091095:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:1.0:1787830217.091100:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787830217.091130:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f3b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572667904:192.168.203.129@tcp:250:kworker.0 00000100:00100000:1.0:1787830217.091130:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f39a40 x1874672572668160/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1787830217.091213:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787830217.091281:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830217/real 1787830217] req@ffff8cb999f3b480 x1874672572667904/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830242 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830217.091298:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f3b480 x1874672572667904/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830242 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830217.091309:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830217.091313:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787830217.091318:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f3b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572667904:192.168.203.129@tcp:250:kworker.0 00000100:00100000:0.0:1787830217.091325:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f3b840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572668032:192.168.203.129@tcp:38:kworker.0 00000800:00000100:0.0:1787830217.091349:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787830217.091362:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830217/real 1787830217] req@ffff8cb999f3b840 x1874672572668032/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830242 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830217.091382:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f3b840 x1874672572668032/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830242 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830217.091390:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830217.091394:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787830217.091397:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f3b840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572668032:192.168.203.129@tcp:38:kworker.0 00000100:00100000:0.0:1787830217.091428:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f39a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572668160:192.168.203.129@tcp:38:kworker.0 00000800:00000100:0.0:1787830217.091450:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787830217.091462:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830217/real 1787830217] req@ffff8cb999f39a40 x1874672572668160/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830242 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830217.091472:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f39a40 x1874672572668160/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830242 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830217.091480:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830217.091483:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787830217.091486:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f39a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572668160:192.168.203.129@tcp:38:kworker.0 00010000:00010000:2.0:1787830217.922976:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787830218.946671:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787830219.971243:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787830220.994462:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787830222.018672:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787830222.210402:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787830222.210413:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787830222.210417:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787830222.210426:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 3174 00000100:00080000:1.0:1787830222.210437:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787830222.210444:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:1.0:1787830222.210478:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f39e00 x1874672572668288/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787830222.210505:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1787830222.210552:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f39e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572668288:192.168.203.129@tcp:250:kworker.0 00000800:00000100:0.0:1787830222.210610:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00080000:1.0:1787830222.210635:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787830222.210637:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787830222.210643:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 3174 00000100:00080000:1.0:1787830222.210648:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1787830222.210649:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830222/real 1787830222] req@ffff8cb999f39e00 x1874672572668288/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830247 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830222.210657:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:1.0:1787830222.210671:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787830222.210676:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f39e00 x1874672572668288/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830247 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830222.210690:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f39a40 x1874672572668416/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787830222.210706:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787830222.210711:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830222.210711:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1787830222.210715:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787830222.210719:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787830222.210725:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f39e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572668288:192.168.203.129@tcp:250:kworker.0 00000100:00080000:1.0:1787830222.210730:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 3174 00000100:00080000:1.0:1787830222.210736:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787830222.210740:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:1.0:1787830222.210746:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787830222.210753:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f39a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572668416:192.168.203.129@tcp:38:kworker.0 00000100:00100000:1.0:1787830222.210761:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f3b840 x1874672572668544/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1787830222.210782:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787830222.210798:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830222/real 1787830222] req@ffff8cb999f39a40 x1874672572668416/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830247 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830222.210810:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f39a40 x1874672572668416/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830247 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830222.210817:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830222.210820:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787830222.210823:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f39a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572668416:192.168.203.129@tcp:38:kworker.0 00000100:00100000:0.0:1787830222.210834:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f3b840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572668544:192.168.203.129@tcp:38:kworker.0 00000800:00000100:0.0:1787830222.210920:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787830222.210933:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830222/real 1787830222] req@ffff8cb999f3b840 x1874672572668544/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830247 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830222.210944:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f3b840 x1874672572668544/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830247 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830222.210953:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830222.210956:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787830222.210959:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f3b840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572668544:192.168.203.129@tcp:38:kworker.0 00010000:00010000:2.0:1787830223.044640:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787830224.066654:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787830225.090959:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787830226.114508:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787830227.138458:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787830227.330436:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787830227.330444:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787830227.330448:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787830227.330456:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 3179 00000100:00080000:2.0:1787830227.330465:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787830227.330476:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:2.0:1787830227.330504:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8ed00 x1874672572668672/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787830227.330524:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787830227.330527:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787830227.330529:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787830227.330532:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 3179 00000100:00080000:2.0:1787830227.330535:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787830227.330539:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:2.0:1787830227.330548:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787830227.330574:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8e940 x1874672572668800/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787830227.330583:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787830227.330586:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1787830227.330588:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787830227.330591:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 3179 00000100:00080000:2.0:1787830227.330601:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787830227.330605:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:2.0:1787830227.330610:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787830227.330622:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8f840 x1874672572668928/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787830227.330989:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8ed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572668672:192.168.203.129@tcp:250:kworker.0 00000800:00000100:0.0:1787830227.331047:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787830227.331098:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830227/real 1787830227] req@ffff8cb998a8ed00 x1874672572668672/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830252 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830227.331115:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8ed00 x1874672572668672/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830252 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830227.331126:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830227.331130:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787830227.331134:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8ed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572668672:192.168.203.129@tcp:250:kworker.0 00000100:00100000:0.0:1787830227.331141:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8e940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572668800:192.168.203.129@tcp:38:kworker.0 00000800:00000100:0.0:1787830227.331176:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787830227.331189:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830227/real 1787830227] req@ffff8cb998a8e940 x1874672572668800/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830252 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830227.331199:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8e940 x1874672572668800/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830252 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830227.331207:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830227.331211:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787830227.331214:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8e940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572668800:192.168.203.129@tcp:38:kworker.0 00000100:00100000:0.0:1787830227.331219:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8f840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572668928:192.168.203.129@tcp:38:kworker.0 00000800:00000100:0.0:1787830227.331236:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787830227.331245:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830227/real 1787830227] req@ffff8cb998a8f840 x1874672572668928/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830252 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830227.331287:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8f840 x1874672572668928/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830252 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830227.331296:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830227.331299:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787830227.331302:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8f840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572668928:192.168.203.129@tcp:38:kworker.0 00010000:00010000:1.0:1787830228.162434:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787830229.186717:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787830230.210543:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787830231.239845:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787830232.258958:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787830232.451093:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787830232.451104:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787830232.451108:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787830232.451119:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 3184 00000100:00080000:2.0:1787830232.451131:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787830232.451138:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:2.0:1787830232.451175:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8f840 x1874672572669056/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787830232.451211:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787830232.451216:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787830232.451232:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787830232.451237:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 3184 00000100:00080000:2.0:1787830232.451241:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787830232.451246:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:0.0:1787830232.451340:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8f840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572669056:192.168.203.129@tcp:250:kworker.0 00000800:00000100:0.0:1787830232.451402:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787830232.451444:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830232/real 1787830232] req@ffff8cb998a8f840 x1874672572669056/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830257 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830232.451462:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8f840 x1874672572669056/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830257 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830232.451473:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830232.451477:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787830232.451480:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8f840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572669056:192.168.203.129@tcp:250:kworker.0 00000008:00100000:2.0:1787830232.451905:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787830232.451923:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8e940 x1874672572669184/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787830232.451944:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787830232.451948:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1787830232.451950:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787830232.451954:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 3184 00000100:00080000:2.0:1787830232.451958:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787830232.451962:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:2.0:1787830232.451968:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787830232.451978:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8ed00 x1874672572669312/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787830232.452769:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8e940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572669184:192.168.203.129@tcp:38:kworker.0 00000800:00000100:0.0:1787830232.452808:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787830232.452828:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830232/real 1787830232] req@ffff8cb998a8e940 x1874672572669184/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830257 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830232.452843:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8e940 x1874672572669184/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830257 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830232.452867:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830232.452871:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787830232.452875:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8e940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572669184:192.168.203.129@tcp:38:kworker.0 00000100:00100000:0.0:1787830232.452882:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8ed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572669312:192.168.203.129@tcp:38:kworker.0 00000800:00000100:0.0:1787830232.452904:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787830232.452915:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830232/real 1787830232] req@ffff8cb998a8ed00 x1874672572669312/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830257 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830232.452926:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8ed00 x1874672572669312/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830257 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830232.452933:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830232.452937:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787830232.452940:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8ed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572669312:192.168.203.129@tcp:38:kworker.0 00010000:00010000:2.0:1787830233.282495:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787830234.306438:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787830235.330425:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787830236.354456:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787830237.378417:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787830237.570452:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787830237.570463:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787830237.570467:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787830237.570477:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 3189 00000100:00080000:3.0:1787830237.570488:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787830237.570495:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:3.0:1787830237.570531:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb9923161c0 x1874672572669440/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787830237.570561:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787830237.570565:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787830237.570567:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787830237.570585:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 3189 00000100:00080000:3.0:1787830237.570590:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787830237.570594:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:3.0:1787830237.570613:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787830237.570628:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb9923170c0 x1874672572669568/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787830237.570638:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787830237.570641:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1787830237.570643:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787830237.570647:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 3189 00000100:00080000:3.0:1787830237.570651:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787830237.570654:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:3.0:1787830237.570659:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787830237.570674:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb992315a40 x1874672572669696/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787830237.570788:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb9923161c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572669440:192.168.203.129@tcp:250:kworker.0 00000800:00000100:0.0:1787830237.570844:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787830237.570881:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830237/real 1787830237] req@ffff8cb9923161c0 x1874672572669440/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830262 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830237.570897:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb9923161c0 x1874672572669440/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830262 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830237.570907:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830237.570911:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787830237.570921:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb9923161c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572669440:192.168.203.129@tcp:250:kworker.0 00000100:00100000:0.0:1787830237.570928:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb9923170c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572669568:192.168.203.129@tcp:38:kworker.0 00000800:00000100:0.0:1787830237.570949:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787830237.570961:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830237/real 1787830237] req@ffff8cb9923170c0 x1874672572669568/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830262 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830237.570972:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb9923170c0 x1874672572669568/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830262 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830237.570979:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830237.570982:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787830237.570985:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb9923170c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572669568:192.168.203.129@tcp:38:kworker.0 00000100:00100000:0.0:1787830237.570991:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb992315a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572669696:192.168.203.129@tcp:38:kworker.0 00000800:00000100:0.0:1787830237.571023:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787830237.571034:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830237/real 1787830237] req@ffff8cb992315a40 x1874672572669696/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830262 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830237.571045:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb992315a40 x1874672572669696/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830262 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830237.571054:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830237.571057:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787830237.571060:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb992315a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572669696:192.168.203.129@tcp:38:kworker.0 00010000:00010000:2.0:1787830238.402439:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787830239.426676:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787830240.450787:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787830241.474514:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787830242.498481:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787830242.690936:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787830242.690947:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787830242.690951:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787830242.690961:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 3194 00000100:00080000:2.0:1787830242.691002:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787830242.691010:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:2.0:1787830242.691042:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8cb40 x1874672572669824/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787830242.691068:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787830242.691072:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787830242.691075:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787830242.691078:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 3194 00000100:00080000:2.0:1787830242.691082:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787830242.691086:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:2.0:1787830242.691097:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787830242.691105:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8cb40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572669824:192.168.203.129@tcp:250:kworker.0 00000100:00100000:2.0:1787830242.691112:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8c000 x1874672572669952/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787830242.691123:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787830242.691126:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1787830242.691129:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787830242.691132:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 3194 00000100:00080000:2.0:1787830242.691136:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787830242.691140:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:2.0:1787830242.691145:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787830242.691156:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8f480 x1874672572670080/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787830242.691207:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572669952:192.168.203.129@tcp:38:kworker.0 00000100:00100000:0.0:1787830242.691245:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572670080:192.168.203.129@tcp:38:kworker.0 00010000:00010000:1.0:1787830243.522446:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787830244.546579:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787830245.570454:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:1.0:1787830245.762607:0:2384:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.129:988 00000400:00000100:1.0:1787830245.762649:0:2384:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.129@tcp at host 192.168.203.129 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:1.0:1787830245.762671:0:2384:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.129@tcp 00000100:00100000:0.0:1787830245.763864:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830242/real 1787830245] req@ffff8cb998a8cb40 x1874672572669824/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830267 ref 2 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830245.763894:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8cb40 x1874672572669824/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830267 ref 2 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830245.763905:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830245.763911:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787830245.763915:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8cb40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572669824:192.168.203.129@tcp:250:kworker.0 00000800:00000100:1.0:1787830245.764058:0:2384:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.129@tcp 00000800:00000100:1.0:1787830245.764083:0:2384:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.129@tcp 00000100:00100000:0.0:1787830245.764307:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830242/real 1787830245] req@ffff8cb998a8c000 x1874672572669952/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830267 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830245.764323:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8c000 x1874672572669952/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830267 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830245.764332:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830245.764336:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787830245.764340:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572669952:192.168.203.129@tcp:38:kworker.0 00000100:00100000:0.0:1787830245.764346:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830242/real 1787830245] req@ffff8cb998a8f480 x1874672572670080/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830267 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830245.764361:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8f480 x1874672572670080/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830267 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830245.764371:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830245.764380:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787830245.764400:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572670080:192.168.203.129@tcp:38:kworker.0 00010000:00010000:1.0:1787830246.594557:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787830247.618549:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787830247.810963:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787830247.810971:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787830247.810976:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787830247.810985:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 3199 00000100:00080000:0.0:1787830247.810995:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787830247.811002:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:0.0:1787830247.811034:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c68000 x1874672572670208/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787830247.811057:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787830247.811060:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787830247.811062:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787830247.811065:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 3199 00000100:00080000:0.0:1787830247.811068:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787830247.811073:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:0.0:1787830247.811082:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787830247.811096:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c6a580 x1874672572670336/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787830247.811106:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787830247.811110:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1787830247.811112:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787830247.811116:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 3199 00000100:00080000:0.0:1787830247.811119:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1787830247.811126:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c68000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572670208:192.168.203.129@tcp:250:kworker.0 00000100:00080000:0.0:1787830247.811140:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:0.0:1787830247.811146:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787830247.811162:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c6a1c0 x1874672572670464/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1787830247.811192:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787830247.811233:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830247/real 1787830247] req@ffff8cb999c68000 x1874672572670208/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830272 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830247.811258:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c68000 x1874672572670208/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830272 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830247.811291:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830247.811296:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787830247.811299:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c68000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572670208:192.168.203.129@tcp:250:kworker.0 00000100:00100000:1.0:1787830247.811307:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c6a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572670336:192.168.203.129@tcp:38:kworker.0 00000800:00000100:1.0:1787830247.811331:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787830247.811343:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830247/real 1787830247] req@ffff8cb999c6a580 x1874672572670336/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830272 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830247.811354:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c6a580 x1874672572670336/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830272 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830247.811375:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830247.811379:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787830247.811383:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c6a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572670336:192.168.203.129@tcp:38:kworker.0 00000100:00100000:1.0:1787830247.812365:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c6a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572670464:192.168.203.129@tcp:38:kworker.0 00000800:00000100:1.0:1787830247.812497:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787830247.812519:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830247/real 1787830247] req@ffff8cb999c6a1c0 x1874672572670464/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830272 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830247.812534:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c6a1c0 x1874672572670464/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830272 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830247.812560:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830247.812563:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787830247.812567:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c6a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572670464:192.168.203.129@tcp:38:kworker.0 00010000:00010000:2.0:1787830248.642757:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787830249.666457:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787830250.690463:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787830251.714559:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787830252.738538:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787830252.930414:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787830252.930424:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787830252.930428:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787830252.930438:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 3205 00000100:00080000:2.0:1787830252.930449:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787830252.930457:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:2.0:1787830252.930492:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8d680 x1874672572670592/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787830252.930520:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787830252.930524:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787830252.930526:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787830252.930530:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 3205 00000100:00080000:2.0:1787830252.930534:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787830252.930537:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:2.0:1787830252.930548:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787830252.930559:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8e580 x1874672572670720/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787830252.930569:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787830252.930572:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1787830252.930574:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787830252.930577:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 3205 00000100:00080000:2.0:1787830252.930580:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787830252.930584:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:2.0:1787830252.930602:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787830252.930612:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8cf00 x1874672572670848/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787830252.930700:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8d680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572670592:192.168.203.129@tcp:250:kworker.0 00000800:00000100:1.0:1787830252.930749:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787830252.930787:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830252/real 1787830252] req@ffff8cb998a8d680 x1874672572670592/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830277 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830252.930800:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8d680 x1874672572670592/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830277 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830252.930807:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830252.930810:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787830252.930812:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8d680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572670592:192.168.203.129@tcp:250:kworker.0 00000100:00100000:1.0:1787830252.930817:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8e580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572670720:192.168.203.129@tcp:38:kworker.0 00000800:00000100:1.0:1787830252.930830:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787830252.930837:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830252/real 1787830252] req@ffff8cb998a8e580 x1874672572670720/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830277 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830252.930844:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8e580 x1874672572670720/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830277 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830252.930862:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830252.930866:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787830252.930869:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8e580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572670720:192.168.203.129@tcp:38:kworker.0 00000100:00100000:1.0:1787830252.930875:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8cf00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572670848:192.168.203.129@tcp:38:kworker.0 00000800:00000100:1.0:1787830252.930910:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787830252.930927:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830252/real 1787830252] req@ffff8cb998a8cf00 x1874672572670848/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830277 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830252.930938:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8cf00 x1874672572670848/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830277 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830252.930946:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830252.930949:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787830252.930952:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8cf00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572670848:192.168.203.129@tcp:38:kworker.0 00010000:00010000:1.0:1787830253.766202:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787830254.786664:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787830255.811730:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787830256.834480:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787830257.858491:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787830258.050364:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787830258.050374:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787830258.050378:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787830258.050388:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 3210 00000100:00080000:2.0:1787830258.050399:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787830258.050406:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:2.0:1787830258.050444:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8fc00 x1874672572670976/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787830258.050471:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787830258.050475:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787830258.050477:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787830258.050481:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 3210 00000100:00080000:2.0:1787830258.050485:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787830258.050489:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:2.0:1787830258.050498:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787830258.050512:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8c780 x1874672572671104/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787830258.050521:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787830258.050537:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1787830258.050539:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787830258.050544:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 3210 00000100:00080000:2.0:1787830258.050549:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787830258.050553:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:2.0:1787830258.050559:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787830258.050566:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8fc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572670976:192.168.203.129@tcp:250:kworker.0 00000100:00100000:2.0:1787830258.050573:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8f0c0 x1874672572671232/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1787830258.050637:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787830258.050681:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830258/real 1787830258] req@ffff8cb998a8fc00 x1874672572670976/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830283 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830258.050699:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8fc00 x1874672572670976/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830283 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830258.050710:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830258.050720:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787830258.050724:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8fc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572670976:192.168.203.129@tcp:250:kworker.0 00000100:00100000:1.0:1787830258.050731:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8c780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572671104:192.168.203.129@tcp:38:kworker.0 00000800:00000100:1.0:1787830258.050752:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787830258.050764:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830258/real 1787830258] req@ffff8cb998a8c780 x1874672572671104/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830283 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830258.050786:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8c780 x1874672572671104/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830283 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830258.050794:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830258.050797:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787830258.050801:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8c780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572671104:192.168.203.129@tcp:38:kworker.0 00000100:00100000:1.0:1787830258.050822:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8f0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572671232:192.168.203.129@tcp:38:kworker.0 00000800:00000100:1.0:1787830258.050839:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787830258.050849:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830258/real 1787830258] req@ffff8cb998a8f0c0 x1874672572671232/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830283 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830258.050871:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8f0c0 x1874672572671232/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830283 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830258.050880:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830258.050886:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787830258.050890:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8f0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572671232:192.168.203.129@tcp:38:kworker.0 00010000:00010000:1.0:1787830258.882680:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787830259.906453:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787830260.931325:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787830261.954459:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787830262.979535:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787830263.170598:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787830263.170609:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787830263.170613:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787830263.170623:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 3215 00000100:00080000:0.0:1787830263.170633:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787830263.170641:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:0.0:1787830263.170675:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c69680 x1874672572671360/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787830263.170695:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787830263.170697:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787830263.170699:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787830263.170701:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 3215 00000100:00080000:0.0:1787830263.170704:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787830263.170706:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:0.0:1787830263.170713:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787830263.170721:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c692c0 x1874672572671488/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787830263.170727:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787830263.170729:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1787830263.170741:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787830263.170744:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 3215 00000100:00080000:0.0:1787830263.170747:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787830263.170749:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:0.0:1787830263.170752:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787830263.170759:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c69e00 x1874672572671616/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787830263.172185:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c69680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572671360:192.168.203.129@tcp:250:kworker.0 00000800:00000100:1.0:1787830263.172288:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787830263.172334:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830263/real 1787830263] req@ffff8cb999c69680 x1874672572671360/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830288 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830263.172353:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c69680 x1874672572671360/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830288 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830263.172365:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830263.172369:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787830263.172373:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c69680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572671360:192.168.203.129@tcp:250:kworker.0 00000100:00100000:1.0:1787830263.172382:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c692c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572671488:192.168.203.129@tcp:38:kworker.0 00000800:00000100:1.0:1787830263.172405:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787830263.172418:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830263/real 1787830263] req@ffff8cb999c692c0 x1874672572671488/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830288 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830263.172431:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c692c0 x1874672572671488/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830288 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830263.172439:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830263.172443:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787830263.172446:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c692c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572671488:192.168.203.129@tcp:38:kworker.0 00000100:00100000:1.0:1787830263.172464:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c69e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572671616:192.168.203.129@tcp:38:kworker.0 00000800:00000100:1.0:1787830263.172486:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787830263.172499:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830263/real 1787830263] req@ffff8cb999c69e00 x1874672572671616/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830288 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830263.172523:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c69e00 x1874672572671616/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830288 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830263.172531:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830263.172534:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787830263.172538:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c69e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572671616:192.168.203.129@tcp:38:kworker.0 00010000:00010000:2.0:1787830264.002866:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787830265.027059:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787830266.050592:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787830267.074484:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787830268.098508:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787830268.290367:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787830268.290377:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787830268.290381:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787830268.290401:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 3220 00000100:00080000:0.0:1787830268.290411:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787830268.290418:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:0.0:1787830268.290453:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c6b480 x1874672572671744/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787830268.290487:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787830268.290491:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787830268.290493:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787830268.290497:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 3220 00000100:00080000:0.0:1787830268.290501:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787830268.290505:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:0.0:1787830268.290523:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787830268.290531:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c6b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572671744:192.168.203.129@tcp:250:kworker.0 00000100:00100000:0.0:1787830268.290538:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c6bc00 x1874672572671872/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787830268.290570:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787830268.290574:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1787830268.290575:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787830268.290589:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 3220 00000100:00080000:0.0:1787830268.290594:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000800:00000100:1.0:1787830268.290595:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00080000:0.0:1787830268.290608:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:0.0:1787830268.290615:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787830268.290630:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c69a40 x1874672572672000/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787830268.290640:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830268/real 1787830268] req@ffff8cb999c6b480 x1874672572671744/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830293 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830268.290671:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c6b480 x1874672572671744/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830293 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830268.290683:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830268.290687:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787830268.290694:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c6b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572671744:192.168.203.129@tcp:250:kworker.0 00000100:00100000:1.0:1787830268.290718:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c6bc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572671872:192.168.203.129@tcp:38:kworker.0 00000800:00000100:1.0:1787830268.290741:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787830268.290753:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830268/real 1787830268] req@ffff8cb999c6bc00 x1874672572671872/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830293 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830268.290766:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c6bc00 x1874672572671872/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830293 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830268.290781:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830268.290795:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787830268.290799:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c6bc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572671872:192.168.203.129@tcp:38:kworker.0 00000100:00100000:1.0:1787830268.290806:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c69a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572672000:192.168.203.129@tcp:38:kworker.0 00000800:00000100:1.0:1787830268.290827:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787830268.290838:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830268/real 1787830268] req@ffff8cb999c69a40 x1874672572672000/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830293 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830268.290850:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c69a40 x1874672572672000/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830293 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830268.290856:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830268.290859:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787830268.290862:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c69a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572672000:192.168.203.129@tcp:38:kworker.0 00010000:00010000:2.0:1787830269.122755:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787830270.146826:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787830271.171331:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787830272.194434:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787830273.218861:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787830273.410417:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787830273.410427:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787830273.410432:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787830273.410442:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 3225 00000100:00080000:1.0:1787830273.410453:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787830273.410460:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:1.0:1787830273.410493:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f3b480 x1874672572672128/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787830273.410520:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787830273.410533:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787830273.410535:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787830273.410538:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 3225 00000100:00080000:1.0:1787830273.410541:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787830273.410545:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:1.0:1787830273.410553:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787830273.410563:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f3b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572672128:192.168.203.129@tcp:250:kworker.0 00000100:00100000:1.0:1787830273.410564:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f3b0c0 x1874672572672256/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787830273.410578:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787830273.410581:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1787830273.410582:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787830273.410594:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 3225 00000100:00080000:1.0:1787830273.410601:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787830273.410605:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:1.0:1787830273.410609:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787830273.410622:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f38b40 x1874672572672384/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1787830273.410628:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787830273.410671:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830273/real 1787830273] req@ffff8cb999f3b480 x1874672572672128/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830298 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830273.410689:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f3b480 x1874672572672128/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830298 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830273.410698:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830273.410701:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787830273.410704:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f3b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572672128:192.168.203.129@tcp:250:kworker.0 00000100:00100000:0.0:1787830273.410724:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f3b0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572672256:192.168.203.129@tcp:38:kworker.0 00000800:00000100:0.0:1787830273.410741:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787830273.410762:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830273/real 1787830273] req@ffff8cb999f3b0c0 x1874672572672256/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830298 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830273.410777:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f3b0c0 x1874672572672256/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830298 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830273.410784:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830273.410787:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787830273.410789:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f3b0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572672256:192.168.203.129@tcp:38:kworker.0 00000100:00100000:0.0:1787830273.410794:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f38b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572672384:192.168.203.129@tcp:38:kworker.0 00000800:00000100:0.0:1787830273.410808:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787830273.410815:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830273/real 1787830273] req@ffff8cb999f38b40 x1874672572672384/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830298 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830273.410822:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f38b40 x1874672572672384/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830298 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830273.410827:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830273.410830:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787830273.410831:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f38b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572672384:192.168.203.129@tcp:38:kworker.0 00010000:00010000:1.0:1787830274.242431:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787830275.266620:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787830276.290683:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787830277.314664:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787830278.338594:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787830278.531434:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787830278.531445:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787830278.531449:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787830278.531461:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 3230 00000100:00080000:0.0:1787830278.531473:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787830278.531481:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:0.0:1787830278.531520:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c68f00 x1874672572672512/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787830278.531568:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787830278.531572:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787830278.531574:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787830278.531578:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 3230 00000100:00080000:0.0:1787830278.531583:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787830278.531588:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:0.0:1787830278.531598:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787830278.531611:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c6b840 x1874672572672640/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787830278.531621:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787830278.531624:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1787830278.531626:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787830278.531629:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 3230 00000100:00080000:0.0:1787830278.531633:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787830278.531636:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:0.0:1787830278.531641:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787830278.531652:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c69a40 x1874672572672768/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787830278.531739:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c68f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572672512:192.168.203.129@tcp:250:kworker.0 00000800:00000100:1.0:1787830278.531803:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787830278.531843:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830278/real 1787830278] req@ffff8cb999c68f00 x1874672572672512/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830303 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830278.531874:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c68f00 x1874672572672512/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830303 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830278.531884:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830278.531888:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787830278.531892:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c68f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572672512:192.168.203.129@tcp:250:kworker.0 00000100:00100000:1.0:1787830278.531898:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c6b840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572672640:192.168.203.129@tcp:38:kworker.0 00000800:00000100:1.0:1787830278.531921:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787830278.531934:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830278/real 1787830278] req@ffff8cb999c6b840 x1874672572672640/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830303 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830278.531947:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c6b840 x1874672572672640/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830303 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830278.531954:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830278.531957:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787830278.531961:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c6b840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572672640:192.168.203.129@tcp:38:kworker.0 00000100:00100000:1.0:1787830278.531967:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c69a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572672768:192.168.203.129@tcp:38:kworker.0 00000800:00000100:1.0:1787830278.531987:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787830278.531998:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830278/real 1787830278] req@ffff8cb999c69a40 x1874672572672768/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830303 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830278.532010:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c69a40 x1874672572672768/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830303 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830278.532028:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830278.532031:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787830278.532034:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c69a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572672768:192.168.203.129@tcp:38:kworker.0 00010000:00010000:2.0:1787830279.362963:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787830280.386545:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787830281.410926:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787830282.434469:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787830283.460051:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787830283.650821:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787830283.650832:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787830283.650836:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787830283.650845:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 3235 00000100:00080000:2.0:1787830283.650872:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787830283.650880:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:2.0:1787830283.650917:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8e1c0 x1874672572672896/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787830283.650949:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787830283.650953:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787830283.650955:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787830283.650959:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 3235 00000100:00080000:2.0:1787830283.650962:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787830283.650966:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:2.0:1787830283.650976:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787830283.650988:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8da40 x1874672572673024/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787830283.650997:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787830283.651000:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1787830283.651001:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787830283.651004:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 3235 00000100:00080000:2.0:1787830283.651008:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787830283.651011:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:2.0:1787830283.651016:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787830283.651027:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8de00 x1874672572673152/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787830283.651329:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8e1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572672896:192.168.203.129@tcp:250:kworker.0 00000800:00000100:1.0:1787830283.651388:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787830283.651424:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830283/real 1787830283] req@ffff8cb998a8e1c0 x1874672572672896/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830308 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830283.651439:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8e1c0 x1874672572672896/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830308 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830283.651449:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830283.651452:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787830283.651456:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8e1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572672896:192.168.203.129@tcp:250:kworker.0 00000100:00100000:1.0:1787830283.651463:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8da40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572673024:192.168.203.129@tcp:38:kworker.0 00000800:00000100:1.0:1787830283.651485:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787830283.651499:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830283/real 1787830283] req@ffff8cb998a8da40 x1874672572673024/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830308 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830283.651511:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8da40 x1874672572673024/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830308 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830283.651519:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830283.651529:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787830283.651532:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8da40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572673024:192.168.203.129@tcp:38:kworker.0 00000100:00100000:1.0:1787830283.651539:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8de00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572673152:192.168.203.129@tcp:38:kworker.0 00000800:00000100:1.0:1787830283.651561:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787830283.651574:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830283/real 1787830283] req@ffff8cb998a8de00 x1874672572673152/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830308 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830283.651585:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8de00 x1874672572673152/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830308 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830283.651593:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830283.651597:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787830283.651600:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8de00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572673152:192.168.203.129@tcp:38:kworker.0 00010000:00010000:2.0:1787830284.482575:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787830285.507287:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787830286.530713:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787830287.554697:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787830288.578641:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787830288.771692:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787830288.771702:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787830288.771705:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787830288.771712:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 3240 00000100:00080000:3.0:1787830288.771721:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787830288.771728:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:3.0:1787830288.771760:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb992316940 x1874672572673280/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787830288.771805:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787830288.771810:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787830288.771812:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787830288.771825:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 3240 00000100:00080000:3.0:1787830288.771831:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787830288.771932:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:3.0:1787830288.771941:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787830288.771954:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb992315e00 x1874672572673408/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787830288.771983:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb992316940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572673280:192.168.203.129@tcp:250:kworker.0 00000800:00000100:1.0:1787830288.772047:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787830288.772081:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830288/real 1787830288] req@ffff8cb992316940 x1874672572673280/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830313 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830288.772097:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb992316940 x1874672572673280/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830313 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787830288.772106:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787830288.772108:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787830288.772115:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1787830288.772116:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00080000:3.0:1787830288.772119:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1787830288.772122:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb992316940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572673280:192.168.203.129@tcp:250:kworker.0 00000100:00100000:1.0:1787830288.772130:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb992315e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572673408:192.168.203.129@tcp:38:kworker.0 00000100:00080000:3.0:1787830288.772136:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 3240 00000100:00080000:3.0:1787830288.772145:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787830288.772150:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:3.0:1787830288.772160:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000800:00000100:1.0:1787830288.772161:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:3.0:1787830288.772180:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb992314f00 x1874672572673536/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787830288.772244:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830288/real 1787830288] req@ffff8cb992315e00 x1874672572673408/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830313 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830288.772257:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb992315e00 x1874672572673408/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830313 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830288.772294:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830288.772297:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787830288.772301:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb992315e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572673408:192.168.203.129@tcp:38:kworker.0 00000100:00100000:1.0:1787830288.772325:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb992314f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572673536:192.168.203.129@tcp:38:kworker.0 00000800:00000100:1.0:1787830288.772345:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787830288.772358:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830288/real 1787830288] req@ffff8cb992314f00 x1874672572673536/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830313 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830288.772370:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb992314f00 x1874672572673536/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830313 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830288.772377:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830288.772381:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787830288.772384:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb992314f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572673536:192.168.203.129@tcp:38:kworker.0 00010000:00010000:1.0:1787830289.602504:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787830290.626746:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787830291.650401:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787830292.675035:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787830293.698473:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787830293.890387:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787830293.890398:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787830293.890401:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787830293.890411:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 3246 00000100:00080000:2.0:1787830293.890421:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787830293.890429:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:2.0:1787830293.890465:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8c3c0 x1874672572673664/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787830293.890496:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787830293.890500:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787830293.890502:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787830293.890505:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 3246 00000100:00080000:2.0:1787830293.890509:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787830293.890512:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:2.0:1787830293.890521:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787830293.890534:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8d2c0 x1874672572673792/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787830293.890543:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787830293.890547:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1787830293.890548:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787830293.890552:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 3246 00000100:00080000:2.0:1787830293.890555:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787830293.890558:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:2.0:1787830293.890577:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787830293.890589:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8de00 x1874672572673920/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787830293.890812:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8c3c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572673664:192.168.203.129@tcp:250:kworker.0 00000800:00000100:1.0:1787830293.890885:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787830293.890924:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830293/real 1787830293] req@ffff8cb998a8c3c0 x1874672572673664/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830318 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830293.890941:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8c3c0 x1874672572673664/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830318 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830293.890950:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830293.890954:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787830293.890957:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8c3c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572673664:192.168.203.129@tcp:250:kworker.0 00000100:00100000:1.0:1787830293.890963:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8d2c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572673792:192.168.203.129@tcp:38:kworker.0 00000800:00000100:1.0:1787830293.890993:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787830293.891014:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830293/real 1787830293] req@ffff8cb998a8d2c0 x1874672572673792/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830318 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830293.891027:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8d2c0 x1874672572673792/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830318 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830293.891035:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830293.891038:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787830293.891041:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8d2c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572673792:192.168.203.129@tcp:38:kworker.0 00000100:00100000:1.0:1787830293.891047:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8de00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572673920:192.168.203.129@tcp:38:kworker.0 00000800:00000100:1.0:1787830293.891067:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787830293.891078:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830293/real 1787830293] req@ffff8cb998a8de00 x1874672572673920/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830318 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830293.891103:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8de00 x1874672572673920/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830318 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830293.891111:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830293.891114:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787830293.891117:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8de00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572673920:192.168.203.129@tcp:38:kworker.0 00010000:00010000:3.0:1787830294.722402:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787830295.746487:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787830296.770484:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787830297.794533:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787830298.818710:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787830299.010537:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787830299.010548:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787830299.010552:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787830299.010562:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 3251 00000100:00080000:0.0:1787830299.010601:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787830299.010608:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:0.0:1787830299.010641:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c6bc00 x1874672572674048/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787830299.010672:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787830299.010676:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787830299.010678:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787830299.010682:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 3251 00000100:00080000:0.0:1787830299.010685:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787830299.010689:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:0.0:1787830299.010698:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787830299.010710:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c6b480 x1874672572674176/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787830299.010719:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787830299.010723:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1787830299.010725:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787830299.010728:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 3251 00000100:00080000:0.0:1787830299.010732:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787830299.010735:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:0.0:1787830299.010739:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787830299.010749:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c69e00 x1874672572674304/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787830299.010875:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c6bc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572674048:192.168.203.129@tcp:250:kworker.0 00000800:00000100:1.0:1787830299.010935:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787830299.010973:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830299/real 1787830299] req@ffff8cb999c6bc00 x1874672572674048/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830324 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830299.010989:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c6bc00 x1874672572674048/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830324 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830299.010999:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830299.011002:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787830299.011006:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c6bc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572674048:192.168.203.129@tcp:250:kworker.0 00000100:00100000:1.0:1787830299.011013:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c6b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572674176:192.168.203.129@tcp:38:kworker.0 00000800:00000100:1.0:1787830299.011143:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787830299.011156:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830299/real 1787830299] req@ffff8cb999c6b480 x1874672572674176/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830324 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830299.011181:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c6b480 x1874672572674176/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830324 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830299.011189:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830299.011192:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787830299.011195:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c6b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572674176:192.168.203.129@tcp:38:kworker.0 00000100:00100000:1.0:1787830299.011200:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c69e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572674304:192.168.203.129@tcp:38:kworker.0 00000800:00000100:1.0:1787830299.011221:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787830299.011233:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830299/real 1787830299] req@ffff8cb999c69e00 x1874672572674304/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830324 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830299.011244:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c69e00 x1874672572674304/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830324 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830299.011251:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830299.011254:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787830299.011258:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c69e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572674304:192.168.203.129@tcp:38:kworker.0 00010000:00010000:1.0:1787830299.843231:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787830300.866390:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787830301.890436:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787830302.915142:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787830303.938645:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787830304.130556:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787830304.130581:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787830304.130591:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787830304.130600:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 3256 00000100:00080000:0.0:1787830304.130610:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787830304.130618:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:0.0:1787830304.130650:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c692c0 x1874672572674432/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787830304.130676:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787830304.130680:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787830304.130681:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787830304.130685:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 3256 00000100:00080000:0.0:1787830304.130689:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787830304.130692:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:0.0:1787830304.130701:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787830304.130713:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c69680 x1874672572674560/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787830304.130724:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787830304.130727:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1787830304.130729:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1787830304.130733:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c692c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572674432:192.168.203.129@tcp:250:kworker.0 00000100:00080000:0.0:1787830304.130734:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 3256 00000100:00080000:0.0:1787830304.130739:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787830304.130743:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:0.0:1787830304.130750:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787830304.130763:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c6a1c0 x1874672572674688/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1787830304.130821:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787830304.130862:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830304/real 1787830304] req@ffff8cb999c692c0 x1874672572674432/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830329 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830304.130892:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c692c0 x1874672572674432/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830329 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830304.130901:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830304.130905:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787830304.130909:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c692c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572674432:192.168.203.129@tcp:250:kworker.0 00000100:00100000:1.0:1787830304.130933:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c69680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572674560:192.168.203.129@tcp:38:kworker.0 00000800:00000100:1.0:1787830304.130956:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787830304.130968:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830304/real 1787830304] req@ffff8cb999c69680 x1874672572674560/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830329 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830304.130979:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c69680 x1874672572674560/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830329 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830304.130988:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830304.130991:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787830304.130994:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c69680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572674560:192.168.203.129@tcp:38:kworker.0 00000100:00100000:1.0:1787830304.131000:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c6a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572674688:192.168.203.129@tcp:38:kworker.0 00000800:00000100:1.0:1787830304.131017:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787830304.131028:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830304/real 1787830304] req@ffff8cb999c6a1c0 x1874672572674688/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830329 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830304.131039:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c6a1c0 x1874672572674688/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830329 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830304.131058:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830304.131062:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787830304.131065:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c6a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572674688:192.168.203.129@tcp:38:kworker.0 00010000:00010000:1.0:1787830304.963337:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787830305.986429:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787830307.010396:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787830308.035319:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787830309.058554:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787830309.250376:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787830309.250387:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787830309.250392:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787830309.250403:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 3261 00000100:00080000:0.0:1787830309.250416:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787830309.250423:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:0.0:1787830309.250458:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c6a580 x1874672572674816/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787830309.250478:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787830309.250481:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787830309.250490:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787830309.250494:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 3261 00000100:00080000:0.0:1787830309.250514:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787830309.250519:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:1.0:1787830309.250524:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c6a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572674816:192.168.203.129@tcp:250:kworker.0 00000008:00100000:0.0:1787830309.250530:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787830309.250546:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c68000 x1874672572674944/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787830309.250557:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787830309.250561:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1787830309.250563:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787830309.250567:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 3261 00000100:00080000:0.0:1787830309.250579:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787830309.250583:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:0.0:1787830309.250601:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787830309.250616:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c6a1c0 x1874672572675072/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787830309.250691:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c68000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572674944:192.168.203.129@tcp:38:kworker.0 00000100:00100000:1.0:1787830309.250718:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c6a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572675072:192.168.203.129@tcp:38:kworker.0 00010000:00010000:3.0:1787830310.082419:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787830311.106414:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787830312.130425:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:1.0:1787830312.322646:0:2381:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.129:988 00000400:00000100:1.0:1787830312.322685:0:2381:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.129@tcp at host 192.168.203.129 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:1.0:1787830312.322702:0:2381:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.129@tcp 00000100:00100000:2.0:1787830312.322930:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830309/real 1787830312] req@ffff8cb999c6a580 x1874672572674816/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830334 ref 2 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787830312.322955:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c6a580 x1874672572674816/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830334 ref 2 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787830312.322966:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787830312.322971:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787830312.322975:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c6a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572674816:192.168.203.129@tcp:250:kworker.0 00000800:00000100:1.0:1787830312.323032:0:2381:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.129@tcp 00000100:00100000:2.0:1787830312.323100:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830309/real 1787830312] req@ffff8cb999c68000 x1874672572674944/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830334 ref 2 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787830312.323119:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c68000 x1874672572674944/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830334 ref 2 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787830312.323127:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787830312.323132:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787830312.323137:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c68000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572674944:192.168.203.129@tcp:38:kworker.0 00000800:00000100:1.0:1787830312.323194:0:2381:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.129@tcp 00000100:00100000:2.0:1787830312.323289:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830309/real 1787830312] req@ffff8cb999c6a1c0 x1874672572675072/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830334 ref 2 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787830312.323323:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c6a1c0 x1874672572675072/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830334 ref 2 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787830312.323331:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787830312.323335:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787830312.323338:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c6a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572675072:192.168.203.129@tcp:38:kworker.0 00010000:00010000:3.0:1787830313.154547:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787830314.178428:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787830314.370358:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787830314.370369:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787830314.370373:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787830314.370382:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 3266 00000100:00080000:0.0:1787830314.370392:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787830314.370398:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:0.0:1787830314.370431:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c69680 x1874672572675200/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787830314.370457:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787830314.370461:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787830314.370463:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787830314.370477:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 3266 00000100:00080000:0.0:1787830314.370481:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787830314.370485:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:0.0:1787830314.370494:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787830314.370507:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c692c0 x1874672572675328/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787830314.370514:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c69680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572675200:192.168.203.129@tcp:250:kworker.0 00000100:00080000:0.0:1787830314.370521:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787830314.370526:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1787830314.370527:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787830314.370531:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 3266 00000100:00080000:0.0:1787830314.370549:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787830314.370554:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:0.0:1787830314.370559:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787830314.370571:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c69e00 x1874672572675456/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1787830314.370581:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787830314.370829:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830314/real 1787830314] req@ffff8cb999c69680 x1874672572675200/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830339 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787830314.370847:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c69680 x1874672572675200/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830339 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787830314.370858:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787830314.370862:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787830314.370866:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c69680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572675200:192.168.203.129@tcp:250:kworker.0 00000100:00100000:2.0:1787830314.370886:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c692c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572675328:192.168.203.129@tcp:38:kworker.0 00000800:00000100:2.0:1787830314.370908:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787830314.370920:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830314/real 1787830314] req@ffff8cb999c692c0 x1874672572675328/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830339 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787830314.370931:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c692c0 x1874672572675328/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830339 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787830314.370952:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787830314.370955:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787830314.370958:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c692c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572675328:192.168.203.129@tcp:38:kworker.0 00000100:00100000:2.0:1787830314.370965:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c69e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572675456:192.168.203.129@tcp:38:kworker.0 00000800:00000100:2.0:1787830314.370984:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787830314.370995:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830314/real 1787830314] req@ffff8cb999c69e00 x1874672572675456/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830339 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787830314.371005:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c69e00 x1874672572675456/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830339 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787830314.371013:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787830314.371016:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787830314.371019:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c69e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572675456:192.168.203.129@tcp:38:kworker.0 00010000:00010000:2.0:1787830315.202990:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787830316.226409:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787830317.250757:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787830318.276040:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787830319.300125:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787830319.490368:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787830319.490383:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787830319.490387:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787830319.490397:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 3271 00000100:00080000:0.0:1787830319.490407:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787830319.490415:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:0.0:1787830319.490458:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c6b480 x1874672572675584/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787830319.490484:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787830319.490486:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787830319.490488:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787830319.490490:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 3271 00000100:00080000:0.0:1787830319.490493:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787830319.490496:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:0.0:1787830319.490503:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787830319.490510:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c6bc00 x1874672572675712/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787830319.490516:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787830319.490529:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1787830319.490531:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787830319.490533:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 3271 00000100:00080000:0.0:1787830319.490536:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787830319.490538:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:0.0:1787830319.490541:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787830319.491537:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c69a40 x1874672572675840/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787830319.492223:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c6b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572675584:192.168.203.129@tcp:250:kworker.0 00000800:00000100:1.0:1787830319.492318:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787830319.492357:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830319/real 1787830319] req@ffff8cb999c6b480 x1874672572675584/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830344 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830319.492374:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c6b480 x1874672572675584/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830344 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830319.492399:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830319.492404:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787830319.492408:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c6b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572675584:192.168.203.129@tcp:250:kworker.0 00000100:00100000:1.0:1787830319.492416:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c6bc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572675712:192.168.203.129@tcp:38:kworker.0 00000800:00000100:1.0:1787830319.493259:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787830319.493297:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830319/real 1787830319] req@ffff8cb999c6bc00 x1874672572675712/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830344 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830319.493313:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c6bc00 x1874672572675712/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830344 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830319.493323:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830319.493327:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787830319.493330:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c6bc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572675712:192.168.203.129@tcp:38:kworker.0 00000100:00100000:1.0:1787830319.493338:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c69a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572675840:192.168.203.129@tcp:38:kworker.0 00000800:00000100:1.0:1787830319.493360:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787830319.493372:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830319/real 1787830319] req@ffff8cb999c69a40 x1874672572675840/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830344 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830319.493383:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c69a40 x1874672572675840/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830344 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830319.493390:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830319.493393:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787830319.493396:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c69a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572675840:192.168.203.129@tcp:38:kworker.0 00010000:00010000:2.0:1787830320.324213:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787830321.354289:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787830322.373148:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787830323.394953:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787830324.418515:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787830324.610699:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787830324.610711:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787830324.610715:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787830324.610724:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 3276 00000100:00080000:0.0:1787830324.610734:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787830324.610741:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:0.0:1787830324.610778:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c6b840 x1874672572675968/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787830324.610811:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787830324.610816:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787830324.610818:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787830324.610822:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 3276 00000100:00080000:0.0:1787830324.610825:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787830324.610829:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:0.0:1787830324.610839:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787830324.610850:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c68f00 x1874672572676096/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787830324.610873:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787830324.610876:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1787830324.612501:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787830324.612514:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 3276 00000100:00080000:0.0:1787830324.612522:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787830324.612528:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:0.0:1787830324.612537:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787830324.612561:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c69a40 x1874672572676224/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787830324.612729:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c6b840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572675968:192.168.203.129@tcp:250:kworker.0 00000800:00000100:1.0:1787830324.612787:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787830324.612826:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830324/real 1787830324] req@ffff8cb999c6b840 x1874672572675968/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830349 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830324.612841:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c6b840 x1874672572675968/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830349 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830324.612851:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830324.612855:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787830324.612859:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c6b840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572675968:192.168.203.129@tcp:250:kworker.0 00000100:00100000:1.0:1787830324.612866:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c68f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572676096:192.168.203.129@tcp:38:kworker.0 00000800:00000100:1.0:1787830324.612902:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787830324.612915:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830324/real 1787830324] req@ffff8cb999c68f00 x1874672572676096/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830349 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830324.612926:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c68f00 x1874672572676096/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830349 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830324.612934:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830324.612938:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787830324.612941:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c68f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572676096:192.168.203.129@tcp:38:kworker.0 00000100:00100000:1.0:1787830324.612947:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c69a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572676224:192.168.203.129@tcp:38:kworker.0 00000800:00000100:1.0:1787830324.612981:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787830324.612993:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830324/real 1787830324] req@ffff8cb999c69a40 x1874672572676224/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830349 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830324.613004:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c69a40 x1874672572676224/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830349 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830324.613093:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830324.613098:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787830324.613101:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c69a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572676224:192.168.203.129@tcp:38:kworker.0 00010000:00010000:3.0:1787830325.442482:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787830326.468117:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787830327.490433:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787830328.514395:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787830329.538512:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787830329.730381:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787830329.730391:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787830329.730395:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787830329.730405:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 3281 00000100:00080000:0.0:1787830329.730416:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787830329.730423:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:0.0:1787830329.730467:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c6bc00 x1874672572676352/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787830329.730497:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787830329.730501:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787830329.730503:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787830329.730506:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 3281 00000100:00080000:0.0:1787830329.730510:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787830329.730513:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:0.0:1787830329.730535:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787830329.730549:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c6b480 x1874672572676480/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787830329.730558:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c6bc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572676352:192.168.203.129@tcp:250:kworker.0 00000100:00080000:0.0:1787830329.730560:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787830329.730563:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1787830329.730566:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787830329.730572:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 3281 00000100:00080000:0.0:1787830329.730575:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787830329.730579:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:0.0:1787830329.730583:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787830329.730596:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c69e00 x1874672572676608/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1787830329.730753:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787830329.730801:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830329/real 1787830329] req@ffff8cb999c6bc00 x1874672572676352/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830354 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830329.730911:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c6bc00 x1874672572676352/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830354 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830329.730922:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830329.730925:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787830329.730929:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c6bc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572676352:192.168.203.129@tcp:250:kworker.0 00000100:00100000:1.0:1787830329.731251:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c6b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572676480:192.168.203.129@tcp:38:kworker.0 00000800:00000100:1.0:1787830329.731303:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787830329.731319:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830329/real 1787830329] req@ffff8cb999c6b480 x1874672572676480/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830354 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830329.731345:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c6b480 x1874672572676480/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830354 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830329.731351:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830329.731353:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787830329.731355:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c6b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572676480:192.168.203.129@tcp:38:kworker.0 00000100:00100000:1.0:1787830329.731372:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c69e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572676608:192.168.203.129@tcp:38:kworker.0 00000800:00000100:1.0:1787830329.731397:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787830329.731414:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830329/real 1787830329] req@ffff8cb999c69e00 x1874672572676608/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830354 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830329.731425:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c69e00 x1874672572676608/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830354 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830329.731433:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830329.731436:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787830329.731439:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c69e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572676608:192.168.203.129@tcp:38:kworker.0 00010000:00010000:1.0:1787830330.562454:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787830331.586451:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787830332.610728:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787830333.634812:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787830334.658651:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787830334.850442:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787830334.850452:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787830334.850457:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787830334.850466:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 3287 00000100:00080000:2.0:1787830334.850493:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787830334.850505:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:2.0:1787830334.850537:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8da40 x1874672572676736/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787830334.850609:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8da40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572676736:192.168.203.129@tcp:250:kworker.0 00000800:00000100:1.0:1787830334.850663:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00080000:2.0:1787830334.850693:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787830334.850698:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787830334.850700:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787830334.850704:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 3287 00000100:00080000:2.0:1787830334.850708:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787830334.850713:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:2.0:1787830334.850722:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787830334.850734:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8e1c0 x1874672572676864/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787830334.852589:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830334/real 1787830334] req@ffff8cb998a8da40 x1874672572676736/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830359 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830334.852613:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8da40 x1874672572676736/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830359 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830334.852633:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830334.852638:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787830334.852641:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8da40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572676736:192.168.203.129@tcp:250:kworker.0 00000100:00100000:1.0:1787830334.852667:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8e1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572676864:192.168.203.129@tcp:38:kworker.0 00000800:00000100:1.0:1787830334.852711:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787830334.852726:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830334/real 1787830334] req@ffff8cb998a8e1c0 x1874672572676864/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830359 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830334.852738:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8e1c0 x1874672572676864/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830359 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830334.852747:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830334.852750:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787830334.852753:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8e1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572676864:192.168.203.129@tcp:38:kworker.0 00000100:00080000:2.0:1787830334.854367:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787830334.854374:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1787830334.854377:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787830334.854386:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 3287 00000100:00080000:2.0:1787830334.854394:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787830334.854401:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:2.0:1787830334.854412:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787830334.854438:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8f0c0 x1874672572676992/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787830334.854480:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8f0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572676992:192.168.203.129@tcp:38:kworker.0 00000800:00000100:1.0:1787830334.854524:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787830334.854544:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830334/real 1787830334] req@ffff8cb998a8f0c0 x1874672572676992/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830359 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830334.854555:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8f0c0 x1874672572676992/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830359 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830334.854562:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830334.854564:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787830334.854567:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8f0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572676992:192.168.203.129@tcp:38:kworker.0 00010000:00010000:2.0:1787830335.684503:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787830336.708902:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787830337.730442:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787830338.754417:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787830339.778585:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787830339.970588:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787830339.970597:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787830339.970601:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787830339.970610:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 3292 00000100:00080000:0.0:1787830339.970620:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787830339.970628:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:0.0:1787830339.970664:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c692c0 x1874672572677120/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787830339.970735:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c692c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572677120:192.168.203.129@tcp:250:kworker.0 00000800:00000100:1.0:1787830339.970783:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00080000:0.0:1787830339.971460:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787830339.971470:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787830339.971472:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787830339.971478:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 3292 00000100:00080000:0.0:1787830339.971497:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787830339.971501:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:0.0:1787830339.971509:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787830339.971522:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c69680 x1874672572677248/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787830339.971604:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830339/real 1787830339] req@ffff8cb999c692c0 x1874672572677120/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830364 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830339.971628:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c692c0 x1874672572677120/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830364 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830339.971637:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830339.971641:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787830339.971645:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c692c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572677120:192.168.203.129@tcp:250:kworker.0 00000100:00100000:1.0:1787830339.971668:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c69680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572677248:192.168.203.129@tcp:38:kworker.0 00000800:00000100:1.0:1787830339.971698:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787830339.971713:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830339/real 1787830339] req@ffff8cb999c69680 x1874672572677248/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830364 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830339.971724:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c69680 x1874672572677248/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830364 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830339.971732:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830339.971735:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787830339.971738:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c69680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572677248:192.168.203.129@tcp:38:kworker.0 00000100:00080000:0.0:1787830339.971839:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787830339.971844:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1787830339.971846:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787830339.971850:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 3292 00000100:00080000:0.0:1787830339.971855:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787830339.971859:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:0.0:1787830339.971865:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787830339.971878:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c6a1c0 x1874672572677376/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787830339.971933:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c6a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572677376:192.168.203.129@tcp:38:kworker.0 00000800:00000100:1.0:1787830339.971968:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787830339.972034:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830339/real 1787830339] req@ffff8cb999c6a1c0 x1874672572677376/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830364 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830339.972047:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c6a1c0 x1874672572677376/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830364 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830339.972054:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830339.972057:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787830339.972061:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c6a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572677376:192.168.203.129@tcp:38:kworker.0 00010000:00010000:3.0:1787830340.802386:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787830341.826429:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787830342.850941:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787830343.874400:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787830344.898803:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787830345.090432:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787830345.090441:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787830345.090444:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787830345.090452:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 3297 00000100:00080000:0.0:1787830345.090461:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787830345.090467:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:0.0:1787830345.090497:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c68000 x1874672572677504/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787830345.090677:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787830345.090692:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787830345.090694:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787830345.090698:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 3297 00000100:00080000:0.0:1787830345.090702:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787830345.090705:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:0.0:1787830345.090715:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787830345.090729:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c6a580 x1874672572677632/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787830345.090739:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787830345.090743:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1787830345.090745:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787830345.090749:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 3297 00000100:00080000:0.0:1787830345.090753:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787830345.090757:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:0.0:1787830345.090761:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787830345.090772:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c6a1c0 x1874672572677760/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787830345.091313:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c68000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572677504:192.168.203.129@tcp:250:kworker.0 00000800:00000100:1.0:1787830345.091370:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787830345.091411:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830345/real 1787830345] req@ffff8cb999c68000 x1874672572677504/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830370 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830345.091429:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c68000 x1874672572677504/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830370 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830345.091440:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830345.091443:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787830345.091447:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c68000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572677504:192.168.203.129@tcp:250:kworker.0 00000100:00100000:1.0:1787830345.091455:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c6a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572677632:192.168.203.129@tcp:38:kworker.0 00000800:00000100:1.0:1787830345.091476:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787830345.091489:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830345/real 1787830345] req@ffff8cb999c6a580 x1874672572677632/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830370 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830345.091515:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c6a580 x1874672572677632/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830370 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830345.091524:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830345.091527:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787830345.091530:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c6a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572677632:192.168.203.129@tcp:38:kworker.0 00000100:00100000:1.0:1787830345.091536:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c6a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572677760:192.168.203.129@tcp:38:kworker.0 00000800:00000100:1.0:1787830345.091558:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787830345.091571:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830345/real 1787830345] req@ffff8cb999c6a1c0 x1874672572677760/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830370 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830345.091583:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c6a1c0 x1874672572677760/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830370 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830345.091590:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830345.091594:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787830345.091597:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c6a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572677760:192.168.203.129@tcp:38:kworker.0 00010000:00010000:1.0:1787830345.922497:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787830346.947704:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787830347.970462:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787830348.995381:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787830350.018420:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787830350.210386:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787830350.210398:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787830350.210402:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787830350.210413:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 3302 00000100:00080000:0.0:1787830350.210424:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787830350.210432:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:0.0:1787830350.210467:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c69680 x1874672572677888/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787830350.210509:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787830350.210512:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787830350.210514:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787830350.210518:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 3302 00000100:00080000:0.0:1787830350.210522:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787830350.210527:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:0.0:1787830350.210536:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787830350.210550:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c692c0 x1874672572678016/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787830350.210557:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c69680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572677888:192.168.203.129@tcp:250:kworker.0 00000100:00080000:0.0:1787830350.210565:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787830350.210569:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1787830350.210571:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787830350.210575:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 3302 00000100:00080000:0.0:1787830350.210579:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787830350.210582:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:0.0:1787830350.210587:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787830350.210599:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c69e00 x1874672572678144/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1787830350.210636:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787830350.210685:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830350/real 1787830350] req@ffff8cb999c69680 x1874672572677888/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830375 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830350.210703:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c69680 x1874672572677888/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830375 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830350.210721:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830350.210725:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787830350.210729:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c69680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572677888:192.168.203.129@tcp:250:kworker.0 00000100:00100000:1.0:1787830350.210751:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c692c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572678016:192.168.203.129@tcp:38:kworker.0 00000800:00000100:1.0:1787830350.210775:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787830350.210792:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830350/real 1787830350] req@ffff8cb999c692c0 x1874672572678016/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830375 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830350.210801:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c692c0 x1874672572678016/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830375 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830350.210806:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830350.210808:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787830350.210810:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c692c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572678016:192.168.203.129@tcp:38:kworker.0 00000100:00100000:1.0:1787830350.210814:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c69e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572678144:192.168.203.129@tcp:38:kworker.0 00000800:00000100:1.0:1787830350.210826:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787830350.210840:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830350/real 1787830350] req@ffff8cb999c69e00 x1874672572678144/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830375 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830350.210847:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c69e00 x1874672572678144/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830375 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830350.210853:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830350.210856:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787830350.210859:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c69e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572678144:192.168.203.129@tcp:38:kworker.0 00010000:00010000:2.0:1787830351.042365:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787830352.066533:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787830353.090457:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787830354.115351:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787830355.138484:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787830355.331894:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787830355.331903:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787830355.331907:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787830355.331923:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 3307 00000100:00080000:0.0:1787830355.331945:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787830355.331953:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:0.0:1787830355.331986:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c6b480 x1874672572678272/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787830355.332021:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787830355.332024:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787830355.332026:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787830355.332030:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 3307 00000100:00080000:0.0:1787830355.332037:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787830355.332041:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:0.0:1787830355.332050:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787830355.332073:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c6bc00 x1874672572678400/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787830355.332082:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787830355.332085:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1787830355.332087:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787830355.332090:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 3307 00000100:00080000:0.0:1787830355.332094:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787830355.332098:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:0.0:1787830355.332109:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787830355.332120:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c69a40 x1874672572678528/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787830355.332446:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c6b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572678272:192.168.203.129@tcp:250:kworker.0 00000800:00000100:1.0:1787830355.332512:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787830355.332549:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830355/real 1787830355] req@ffff8cb999c6b480 x1874672572678272/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830380 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830355.332565:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c6b480 x1874672572678272/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830380 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830355.332575:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830355.332578:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787830355.332582:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c6b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572678272:192.168.203.129@tcp:250:kworker.0 00000100:00100000:1.0:1787830355.332589:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c6bc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572678400:192.168.203.129@tcp:38:kworker.0 00000800:00000100:1.0:1787830355.332609:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787830355.332621:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830355/real 1787830355] req@ffff8cb999c6bc00 x1874672572678400/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830380 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830355.332631:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c6bc00 x1874672572678400/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830380 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830355.332639:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830355.332642:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787830355.332645:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c6bc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572678400:192.168.203.129@tcp:38:kworker.0 00000100:00100000:1.0:1787830355.332651:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c69a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572678528:192.168.203.129@tcp:38:kworker.0 00000800:00000100:1.0:1787830355.332681:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787830355.332692:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830355/real 1787830355] req@ffff8cb999c69a40 x1874672572678528/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830380 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830355.332704:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c69a40 x1874672572678528/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830380 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830355.332712:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830355.332716:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787830355.332720:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c69a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572678528:192.168.203.129@tcp:38:kworker.0 00010000:00010000:3.0:1787830356.162399:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787830357.186446:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787830358.210537:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787830359.235183:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787830360.258491:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787830360.450657:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787830360.450694:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787830360.450699:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787830360.450711:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 3312 00000100:00080000:0.0:1787830360.450724:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787830360.450732:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:0.0:1787830360.450872:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c68f00 x1874672572678656/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787830360.450919:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787830360.450923:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787830360.450924:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787830360.450928:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 3312 00000100:00080000:0.0:1787830360.450931:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787830360.450935:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:0.0:1787830360.450946:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787830360.450956:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c6b840 x1874672572678784/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787830360.450965:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787830360.450967:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1787830360.450969:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787830360.450972:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 3312 00000100:00080000:0.0:1787830360.450975:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787830360.450978:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:0.0:1787830360.450982:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787830360.450992:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c69a40 x1874672572678912/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787830360.452083:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c68f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572678656:192.168.203.129@tcp:250:kworker.0 00000800:00000100:1.0:1787830360.452177:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787830360.452245:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830360/real 1787830360] req@ffff8cb999c68f00 x1874672572678656/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830385 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830360.452293:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c68f00 x1874672572678656/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830385 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830360.452303:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830360.452306:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787830360.452310:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c68f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572678656:192.168.203.129@tcp:250:kworker.0 00000100:00100000:1.0:1787830360.452316:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c6b840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572678784:192.168.203.129@tcp:38:kworker.0 00000800:00000100:1.0:1787830360.452335:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787830360.452355:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830360/real 1787830360] req@ffff8cb999c6b840 x1874672572678784/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830385 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830360.452362:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c6b840 x1874672572678784/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830385 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830360.452368:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830360.452370:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787830360.452372:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c6b840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572678784:192.168.203.129@tcp:38:kworker.0 00000100:00100000:1.0:1787830360.452376:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c69a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572678912:192.168.203.129@tcp:38:kworker.0 00000800:00000100:1.0:1787830360.452388:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787830360.452394:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830360/real 1787830360] req@ffff8cb999c69a40 x1874672572678912/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830385 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830360.452401:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c69a40 x1874672572678912/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830385 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830360.452405:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830360.452406:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787830360.452408:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c69a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572678912:192.168.203.129@tcp:38:kworker.0 00010000:00010000:1.0:1787830361.282453:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787830362.306409:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787830363.330659:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787830364.354453:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787830365.379471:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787830365.570410:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787830365.570559:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787830365.570563:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787830365.570573:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 3317 00000100:00080000:0.0:1787830365.570584:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787830365.570592:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:0.0:1787830365.570630:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c6bc00 x1874672572679040/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787830365.570660:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787830365.570664:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787830365.570666:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787830365.570669:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 3317 00000100:00080000:0.0:1787830365.570673:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787830365.570676:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:0.0:1787830365.570685:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787830365.570698:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c6b480 x1874672572679168/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787830365.570708:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787830365.570711:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1787830365.570713:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787830365.570717:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 3317 00000100:00080000:0.0:1787830365.570720:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787830365.570723:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:0.0:1787830365.570741:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787830365.570752:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c69e00 x1874672572679296/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787830365.570837:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c6bc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572679040:192.168.203.129@tcp:250:kworker.0 00000800:00000100:1.0:1787830365.570906:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787830365.570952:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830365/real 1787830365] req@ffff8cb999c6bc00 x1874672572679040/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830390 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830365.570976:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c6bc00 x1874672572679040/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830390 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830365.570996:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830365.571000:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787830365.571004:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c6bc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572679040:192.168.203.129@tcp:250:kworker.0 00000100:00100000:1.0:1787830365.571011:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c6b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572679168:192.168.203.129@tcp:38:kworker.0 00000800:00000100:1.0:1787830365.571033:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787830365.571046:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830365/real 1787830365] req@ffff8cb999c6b480 x1874672572679168/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830390 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830365.571058:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c6b480 x1874672572679168/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830390 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830365.571066:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830365.571069:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787830365.571072:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c6b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572679168:192.168.203.129@tcp:38:kworker.0 00000100:00100000:1.0:1787830365.571078:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c69e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572679296:192.168.203.129@tcp:38:kworker.0 00000800:00000100:1.0:1787830365.571095:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787830365.571105:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830365/real 1787830365] req@ffff8cb999c69e00 x1874672572679296/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830390 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830365.571128:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c69e00 x1874672572679296/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830390 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830365.571148:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830365.571151:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787830365.571154:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c69e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572679296:192.168.203.129@tcp:38:kworker.0 00010000:00010000:1.0:1787830366.402690:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787830367.427128:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787830368.450379:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787830369.474520:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787830370.499150:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787830370.690420:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787830370.690431:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787830370.690435:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787830370.690447:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 3322 00000100:00080000:1.0:1787830370.690459:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787830370.690466:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:1.0:1787830370.690508:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f38f00 x1874672572679424/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787830370.690533:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787830370.690536:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787830370.690537:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787830370.690539:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 3322 00000100:00080000:1.0:1787830370.690541:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787830370.690544:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:1.0:1787830370.690555:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787830370.690567:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f392c0 x1874672572679552/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787830370.690584:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f38f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572679424:192.168.203.129@tcp:250:kworker.0 00000100:00080000:1.0:1787830370.690612:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787830370.690616:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1787830370.690619:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787830370.690624:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 3322 00000100:00080000:1.0:1787830370.690628:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787830370.690633:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:1.0:1787830370.690639:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787830370.690653:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f3a580 x1874672572679680/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1787830370.690679:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787830370.690775:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830370/real 1787830370] req@ffff8cb999f38f00 x1874672572679424/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830395 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787830370.690793:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f38f00 x1874672572679424/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830395 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787830370.690802:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787830370.690805:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787830370.690809:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f38f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572679424:192.168.203.129@tcp:250:kworker.0 00000100:00100000:2.0:1787830370.690846:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f392c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572679552:192.168.203.129@tcp:38:kworker.0 00000800:00000100:2.0:1787830370.690871:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787830370.690881:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830370/real 1787830370] req@ffff8cb999f392c0 x1874672572679552/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830395 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787830370.690889:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f392c0 x1874672572679552/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830395 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787830370.690894:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787830370.690896:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787830370.690898:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f392c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572679552:192.168.203.129@tcp:38:kworker.0 00000100:00100000:2.0:1787830370.690918:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f3a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572679680:192.168.203.129@tcp:38:kworker.0 00000800:00000100:2.0:1787830370.690933:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787830370.690940:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830370/real 1787830370] req@ffff8cb999f3a580 x1874672572679680/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830395 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787830370.690948:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f3a580 x1874672572679680/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830395 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787830370.690952:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787830370.690954:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787830370.690956:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f3a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572679680:192.168.203.129@tcp:38:kworker.0 00010000:00010000:3.0:1787830371.522422:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787830372.546928:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787830373.571077:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787830374.595212:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787830375.618547:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787830375.810329:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787830375.810340:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787830375.810344:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787830375.810356:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 3327 00000100:00080000:1.0:1787830375.810367:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787830375.810374:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:1.0:1787830375.810410:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f383c0 x1874672572679808/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787830375.810441:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787830375.810445:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787830375.810447:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787830375.810451:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 3327 00000100:00080000:1.0:1787830375.810469:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787830375.810474:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:1.0:1787830375.810484:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787830375.810498:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f3a580 x1874672572679936/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787830375.810509:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787830375.810512:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1787830375.810513:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787830375.810517:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 3327 00000100:00080000:1.0:1787830375.810520:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787830375.810523:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:1.0:1787830375.810528:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787830375.810539:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f392c0 x1874672572680064/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787830375.810616:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f383c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572679808:192.168.203.129@tcp:250:kworker.0 00000100:00100000:2.0:1787830375.810716:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f3a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572679936:192.168.203.129@tcp:38:kworker.0 00000100:00100000:2.0:1787830375.810739:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f392c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572680064:192.168.203.129@tcp:38:kworker.0 00010000:00010000:2.0:1787830376.642467:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787830377.666935:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787830378.691851:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:2.0:1787830378.884080:0:2382:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.129:988 00000400:00000100:2.0:1787830378.884129:0:2382:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.129@tcp at host 192.168.203.129 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:2.0:1787830378.884145:0:2382:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.129@tcp 00000800:00000100:2.0:1787830378.884183:0:2382:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.129@tcp 00000800:00000100:2.0:1787830378.884192:0:2382:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.129@tcp 00000100:00100000:1.0:1787830378.884707:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830375/real 1787830378] req@ffff8cb999f383c0 x1874672572679808/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830400 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830378.885129:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f383c0 x1874672572679808/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830400 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830378.885140:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830378.885143:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787830378.885146:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f383c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572679808:192.168.203.129@tcp:250:kworker.0 00000100:00100000:1.0:1787830378.885153:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830375/real 1787830378] req@ffff8cb999f3a580 x1874672572679936/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830400 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830378.885176:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f3a580 x1874672572679936/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830400 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830378.885182:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830378.885184:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787830378.885186:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f3a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572679936:192.168.203.129@tcp:38:kworker.0 00000100:00100000:1.0:1787830378.885190:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830375/real 1787830378] req@ffff8cb999f392c0 x1874672572680064/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830400 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830378.885196:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f392c0 x1874672572680064/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830400 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830378.885200:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830378.885202:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787830378.885204:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f392c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572680064:192.168.203.129@tcp:38:kworker.0 00010000:00010000:2.0:1787830379.716052:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787830380.738424:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787830380.930421:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787830380.930428:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787830380.930431:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787830380.930439:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 3333 00000100:00080000:0.0:1787830380.930449:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787830380.930456:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:0.0:1787830380.930488:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c692c0 x1874672572680192/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787830380.930514:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787830380.930517:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787830380.930518:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787830380.930521:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 3333 00000100:00080000:0.0:1787830380.930523:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787830380.930525:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:0.0:1787830380.930533:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787830380.930541:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c69680 x1874672572680320/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787830380.930547:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787830380.930549:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1787830380.930550:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787830380.930552:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 3333 00000100:00080000:0.0:1787830380.930565:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787830380.930567:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:0.0:1787830380.930570:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787830380.930576:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c6a1c0 x1874672572680448/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787830380.930629:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c692c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572680192:192.168.203.129@tcp:250:kworker.0 00000800:00000100:1.0:1787830380.930684:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787830380.930720:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830380/real 1787830380] req@ffff8cb999c692c0 x1874672572680192/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830405 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830380.930737:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c692c0 x1874672572680192/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830405 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830380.930746:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830380.930750:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787830380.930760:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c692c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572680192:192.168.203.129@tcp:250:kworker.0 00000100:00100000:1.0:1787830380.930767:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c69680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572680320:192.168.203.129@tcp:38:kworker.0 00000800:00000100:1.0:1787830380.930789:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787830380.930802:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830380/real 1787830380] req@ffff8cb999c69680 x1874672572680320/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830405 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830380.930814:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c69680 x1874672572680320/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830405 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830380.930822:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830380.930825:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787830380.930828:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c69680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572680320:192.168.203.129@tcp:38:kworker.0 00000100:00100000:1.0:1787830380.930834:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c6a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572680448:192.168.203.129@tcp:38:kworker.0 00000800:00000100:1.0:1787830380.930852:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787830380.930862:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830380/real 1787830380] req@ffff8cb999c6a1c0 x1874672572680448/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830405 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830380.930872:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c6a1c0 x1874672572680448/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830405 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830380.930880:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830380.930883:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787830380.930886:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c6a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572680448:192.168.203.129@tcp:38:kworker.0 00010000:00010000:1.0:1787830381.762616:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787830382.786396:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787830383.811400:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787830384.834575:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787830385.864583:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787830386.050506:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787830386.050516:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787830386.050535:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787830386.050545:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 3338 00000100:00080000:2.0:1787830386.050555:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787830386.050563:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:2.0:1787830386.050605:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8c780 x1874672572680576/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787830386.050635:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787830386.050639:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787830386.050641:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787830386.050645:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 3338 00000100:00080000:2.0:1787830386.050648:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787830386.050652:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:1.0:1787830386.050760:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8c780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572680576:192.168.203.129@tcp:250:kworker.0 00000800:00000100:1.0:1787830386.050831:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787830386.050920:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830386/real 1787830386] req@ffff8cb998a8c780 x1874672572680576/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830411 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830386.050943:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8c780 x1874672572680576/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830411 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830386.050953:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830386.050957:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787830386.050961:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8c780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572680576:192.168.203.129@tcp:250:kworker.0 00000008:00100000:2.0:1787830386.051050:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787830386.051066:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8fc00 x1874672572680704/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787830386.051087:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787830386.051091:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1787830386.051093:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787830386.051097:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 3338 00000100:00080000:2.0:1787830386.051101:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787830386.051106:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:2.0:1787830386.051111:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787830386.051122:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8cf00 x1874672572680832/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787830386.051187:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8fc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572680704:192.168.203.129@tcp:38:kworker.0 00000800:00000100:1.0:1787830386.051214:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787830386.051228:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830386/real 1787830386] req@ffff8cb998a8fc00 x1874672572680704/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830411 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830386.051239:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8fc00 x1874672572680704/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830411 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830386.051248:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830386.051251:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787830386.051254:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8fc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572680704:192.168.203.129@tcp:38:kworker.0 00000100:00100000:1.0:1787830386.051286:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8cf00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572680832:192.168.203.129@tcp:38:kworker.0 00000800:00000100:1.0:1787830386.051305:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787830386.051323:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830386/real 1787830386] req@ffff8cb998a8cf00 x1874672572680832/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830411 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830386.051334:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8cf00 x1874672572680832/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830411 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830386.051341:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830386.051343:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787830386.051346:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8cf00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572680832:192.168.203.129@tcp:38:kworker.0 00010000:00010000:3.0:1787830386.883031:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787830387.906482:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787830388.930513:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787830389.954396:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787830390.978752:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787830391.171039:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787830391.171046:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787830391.171050:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787830391.171059:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 3343 00000100:00080000:0.0:1787830391.171069:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787830391.171077:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:0.0:1787830391.171107:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c6a580 x1874672572680960/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787830391.171130:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787830391.171134:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787830391.171136:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787830391.171139:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 3343 00000100:00080000:0.0:1787830391.171143:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787830391.171146:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:0.0:1787830391.171155:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787830391.171166:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c68000 x1874672572681088/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787830391.171176:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787830391.171191:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1787830391.171193:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787830391.171197:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 3343 00000100:00080000:0.0:1787830391.171201:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787830391.171205:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:0.0:1787830391.171210:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787830391.171221:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c6a1c0 x1874672572681216/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787830391.171285:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c6a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572680960:192.168.203.129@tcp:250:kworker.0 00000800:00000100:1.0:1787830391.171342:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787830391.171376:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830391/real 1787830391] req@ffff8cb999c6a580 x1874672572680960/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830416 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830391.171392:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c6a580 x1874672572680960/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830416 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830391.171402:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830391.171406:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787830391.171409:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c6a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572680960:192.168.203.129@tcp:250:kworker.0 00000100:00100000:1.0:1787830391.171416:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c68000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572681088:192.168.203.129@tcp:38:kworker.0 00000800:00000100:1.0:1787830391.171438:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787830391.171449:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830391/real 1787830391] req@ffff8cb999c68000 x1874672572681088/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830416 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830391.171474:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c68000 x1874672572681088/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830416 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830391.171491:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830391.171495:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787830391.171498:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c68000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572681088:192.168.203.129@tcp:38:kworker.0 00000100:00100000:1.0:1787830391.171505:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c6a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572681216:192.168.203.129@tcp:38:kworker.0 00000800:00000100:1.0:1787830391.171526:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787830391.171537:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830391/real 1787830391] req@ffff8cb999c6a1c0 x1874672572681216/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830416 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830391.171549:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c6a1c0 x1874672572681216/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830416 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830391.171556:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830391.171559:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787830391.171563:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c6a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572681216:192.168.203.129@tcp:38:kworker.0 00010000:00010000:1.0:1787830392.002416:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787830393.026401:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787830394.050690:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787830395.074883:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787830396.098410:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787830396.290716:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787830396.290727:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787830396.290730:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787830396.290740:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 3348 00000100:00080000:0.0:1787830396.290749:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787830396.290756:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:0.0:1787830396.290791:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c69680 x1874672572681344/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787830396.290821:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787830396.290824:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787830396.290826:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787830396.290830:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 3348 00000100:00080000:0.0:1787830396.290833:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787830396.290837:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:0.0:1787830396.290846:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787830396.290858:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c692c0 x1874672572681472/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787830396.290880:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787830396.290883:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1787830396.290885:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787830396.290889:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 3348 00000100:00080000:0.0:1787830396.290901:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787830396.290906:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:0.0:1787830396.290912:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787830396.290923:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c69e00 x1874672572681600/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787830396.291302:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c69680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572681344:192.168.203.129@tcp:250:kworker.0 00000800:00000100:1.0:1787830396.291362:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787830396.291415:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830396/real 1787830396] req@ffff8cb999c69680 x1874672572681344/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830421 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830396.291445:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c69680 x1874672572681344/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830421 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830396.291455:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830396.291459:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787830396.291463:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c69680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572681344:192.168.203.129@tcp:250:kworker.0 00000100:00100000:1.0:1787830396.291471:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c692c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572681472:192.168.203.129@tcp:38:kworker.0 00000800:00000100:1.0:1787830396.291494:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787830396.291507:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830396/real 1787830396] req@ffff8cb999c692c0 x1874672572681472/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830421 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830396.291517:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c692c0 x1874672572681472/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830421 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830396.291535:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830396.291538:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787830396.291542:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c692c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572681472:192.168.203.129@tcp:38:kworker.0 00000100:00100000:1.0:1787830396.291547:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c69e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572681600:192.168.203.129@tcp:38:kworker.0 00000800:00000100:1.0:1787830396.291567:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787830396.291579:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830396/real 1787830396] req@ffff8cb999c69e00 x1874672572681600/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830421 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830396.291722:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c69e00 x1874672572681600/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830421 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830396.291741:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830396.291744:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787830396.291747:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c69e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572681600:192.168.203.129@tcp:38:kworker.0 00010000:00010000:1.0:1787830397.123285:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787830398.146605:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787830399.171247:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787830400.194417:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787830401.218974:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787830401.411548:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787830401.411558:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787830401.411561:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787830401.411586:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 3353 00000100:00080000:3.0:1787830401.411595:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787830401.411602:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:3.0:1787830401.411634:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb992314780 x1874672572681728/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787830401.411707:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb992314780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572681728:192.168.203.129@tcp:250:kworker.0 00000800:00000100:1.0:1787830401.411763:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00080000:3.0:1787830401.411794:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787830401.411798:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787830401.411801:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787830401.411805:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 3353 00000100:00080000:3.0:1787830401.411809:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787830401.411813:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:3.0:1787830401.411833:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787830401.411848:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb992314b40 x1874672572681856/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787830401.412158:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830401/real 1787830401] req@ffff8cb992314780 x1874672572681728/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830426 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830401.412176:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb992314780 x1874672572681728/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830426 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830401.412186:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830401.412190:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787830401.412194:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb992314780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572681728:192.168.203.129@tcp:250:kworker.0 00000100:00100000:1.0:1787830401.412218:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb992314b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572681856:192.168.203.129@tcp:38:kworker.0 00000800:00000100:1.0:1787830401.412243:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787830401.412257:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830401/real 1787830401] req@ffff8cb992314b40 x1874672572681856/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830426 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830401.412295:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb992314b40 x1874672572681856/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830426 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830401.412316:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830401.412320:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787830401.412324:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb992314b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572681856:192.168.203.129@tcp:38:kworker.0 00000100:00080000:3.0:1787830401.412567:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787830401.412572:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1787830401.412575:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787830401.412580:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 3353 00000100:00080000:3.0:1787830401.412585:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787830401.412591:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:3.0:1787830401.412597:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787830401.412611:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb992317480 x1874672572681984/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787830401.412694:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb992317480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572681984:192.168.203.129@tcp:38:kworker.0 00000800:00000100:1.0:1787830401.412721:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787830401.412735:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830401/real 1787830401] req@ffff8cb992317480 x1874672572681984/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830426 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830401.412746:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb992317480 x1874672572681984/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830426 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830401.412754:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830401.412757:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787830401.412760:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb992317480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572681984:192.168.203.129@tcp:38:kworker.0 00010000:00010000:0.0:1787830402.242461:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787830403.266471:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787830404.290446:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787830405.315563:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787830406.338431:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787830406.530358:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787830406.530368:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787830406.530372:0:37:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787830406.530380:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 3358 00000100:00080000:3.0:1787830406.530390:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787830406.530397:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:3.0:1787830406.530428:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb992316580 x1874672572682112/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787830406.530455:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787830406.530459:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787830406.530461:0:37:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787830406.530464:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 3358 00000100:00080000:3.0:1787830406.530467:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787830406.530470:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:3.0:1787830406.530478:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787830406.530488:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb992317480 x1874672572682240/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787830406.530497:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787830406.530501:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1787830406.530502:0:37:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787830406.530505:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 3358 00000100:00080000:3.0:1787830406.530508:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787830406.530511:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:3.0:1787830406.530526:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787830406.530534:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb992314b40 x1874672572682368/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787830406.530583:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb992316580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572682112:192.168.203.129@tcp:250:kworker.0 00000800:00000100:1.0:1787830406.530644:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787830406.530703:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830406/real 1787830406] req@ffff8cb992316580 x1874672572682112/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830431 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830406.530721:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb992316580 x1874672572682112/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830431 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830406.530745:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830406.530749:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787830406.530753:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb992316580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572682112:192.168.203.129@tcp:250:kworker.0 00000100:00100000:1.0:1787830406.530760:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb992317480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572682240:192.168.203.129@tcp:38:kworker.0 00000800:00000100:1.0:1787830406.530778:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787830406.530787:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830406/real 1787830406] req@ffff8cb992317480 x1874672572682240/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830431 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830406.530794:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb992317480 x1874672572682240/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830431 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830406.530799:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830406.530802:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787830406.530805:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb992317480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572682240:192.168.203.129@tcp:38:kworker.0 00000100:00100000:1.0:1787830406.530811:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb992314b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572682368:192.168.203.129@tcp:38:kworker.0 00000800:00000100:1.0:1787830406.530839:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787830406.530850:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830406/real 1787830406] req@ffff8cb992314b40 x1874672572682368/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830431 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830406.530862:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb992314b40 x1874672572682368/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830431 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830406.530870:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830406.530877:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787830406.530880:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb992314b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572682368:192.168.203.129@tcp:38:kworker.0 00010000:00010000:1.0:1787830407.362477:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787830408.386707:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787830409.413184:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787830410.434417:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787830411.458426:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787830411.650523:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787830411.650530:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787830411.650533:0:37:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787830411.650540:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 3363 00000100:00080000:3.0:1787830411.650548:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787830411.650553:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:3.0:1787830411.650581:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb992314780 x1874672572682496/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787830411.650607:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787830411.650609:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787830411.650611:0:37:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787830411.650613:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 3363 00000100:00080000:3.0:1787830411.650615:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787830411.650617:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:3.0:1787830411.650624:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787830411.650631:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb992314f00 x1874672572682624/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787830411.650636:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787830411.650638:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1787830411.650639:0:37:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787830411.650641:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 3363 00000100:00080000:3.0:1787830411.650655:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787830411.650658:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:3.0:1787830411.650660:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787830411.650666:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb992315e00 x1874672572682752/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787830411.650872:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb992314780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572682496:192.168.203.129@tcp:250:kworker.0 00000800:00000100:1.0:1787830411.650930:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787830411.650969:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830411/real 1787830411] req@ffff8cb992314780 x1874672572682496/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830436 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830411.650985:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb992314780 x1874672572682496/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830436 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830411.650995:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830411.651000:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787830411.651005:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb992314780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572682496:192.168.203.129@tcp:250:kworker.0 00000100:00100000:1.0:1787830411.651013:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb992314f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572682624:192.168.203.129@tcp:38:kworker.0 00000800:00000100:1.0:1787830411.651036:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787830411.651048:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830411/real 1787830411] req@ffff8cb992314f00 x1874672572682624/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830436 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830411.651059:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb992314f00 x1874672572682624/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830436 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830411.651067:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830411.651070:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787830411.651073:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb992314f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572682624:192.168.203.129@tcp:38:kworker.0 00000100:00100000:1.0:1787830411.651079:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb992315e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572682752:192.168.203.129@tcp:38:kworker.0 00000800:00000100:1.0:1787830411.651119:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787830411.651131:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830411/real 1787830411] req@ffff8cb992315e00 x1874672572682752/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830436 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830411.651142:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb992315e00 x1874672572682752/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830436 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830411.651150:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830411.651154:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787830411.651157:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb992315e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572682752:192.168.203.129@tcp:38:kworker.0 00010000:00010000:1.0:1787830412.482593:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787830413.506447:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787830414.530871:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787830415.555920:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787830416.578801:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787830416.770438:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787830416.770449:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787830416.770453:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787830416.770462:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 3368 00000100:00080000:0.0:1787830416.770473:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787830416.770480:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:0.0:1787830416.770514:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c6b480 x1874672572682880/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787830416.770545:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787830416.770549:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787830416.770564:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787830416.770569:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 3368 00000100:00080000:0.0:1787830416.770573:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787830416.770589:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:0.0:1787830416.770599:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787830416.770613:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c6bc00 x1874672572683008/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787830416.770623:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787830416.770626:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1787830416.770628:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787830416.770631:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 3368 00000100:00080000:0.0:1787830416.770635:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787830416.770638:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:0.0:1787830416.770642:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787830416.770653:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c69a40 x1874672572683136/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787830416.770681:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c6b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572682880:192.168.203.129@tcp:250:kworker.0 00000800:00000100:1.0:1787830416.770738:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787830416.770785:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830416/real 1787830416] req@ffff8cb999c6b480 x1874672572682880/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830441 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830416.770802:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c6b480 x1874672572682880/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830441 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830416.770812:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830416.770815:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787830416.770819:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c6b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572682880:192.168.203.129@tcp:250:kworker.0 00000100:00100000:1.0:1787830416.770826:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c6bc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572683008:192.168.203.129@tcp:38:kworker.0 00000800:00000100:1.0:1787830416.770860:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787830416.770873:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830416/real 1787830416] req@ffff8cb999c6bc00 x1874672572683008/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830441 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830416.770886:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c6bc00 x1874672572683008/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830441 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830416.770893:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830416.770897:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787830416.770900:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c6bc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572683008:192.168.203.129@tcp:38:kworker.0 00000100:00100000:1.0:1787830416.770906:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c69a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572683136:192.168.203.129@tcp:38:kworker.0 00000800:00000100:1.0:1787830416.770925:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787830416.770937:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830416/real 1787830416] req@ffff8cb999c69a40 x1874672572683136/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830441 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830416.770948:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c69a40 x1874672572683136/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830441 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830416.770956:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830416.770959:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787830416.770962:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c69a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572683136:192.168.203.129@tcp:38:kworker.0 00010000:00010000:0.0:1787830417.602458:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787830418.626437:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787830419.653434:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787830420.674568:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787830421.698646:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787830421.890352:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787830421.890362:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787830421.890367:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787830421.890377:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 3374 00000100:00080000:0.0:1787830421.890388:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787830421.890394:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:0.0:1787830421.890443:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c6b840 x1874672572683264/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787830421.890475:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787830421.890479:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787830421.890481:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787830421.890485:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 3374 00000100:00080000:0.0:1787830421.890489:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787830421.890494:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:0.0:1787830421.890503:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787830421.890513:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c68f00 x1874672572683392/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787830421.890522:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787830421.890525:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1787830421.890527:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787830421.890530:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 3374 00000100:00080000:0.0:1787830421.890533:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787830421.890537:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:0.0:1787830421.890541:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787830421.890551:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c69a40 x1874672572683520/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787830421.890775:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c6b840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572683264:192.168.203.129@tcp:250:kworker.0 00000800:00000100:1.0:1787830421.890837:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787830421.890891:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830421/real 1787830421] req@ffff8cb999c6b840 x1874672572683264/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830446 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830421.890908:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c6b840 x1874672572683264/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830446 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830421.890918:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830421.890922:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787830421.890925:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c6b840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572683264:192.168.203.129@tcp:250:kworker.0 00000100:00100000:1.0:1787830421.890939:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c68f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572683392:192.168.203.129@tcp:38:kworker.0 00000800:00000100:1.0:1787830421.890962:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787830421.890975:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830421/real 1787830421] req@ffff8cb999c68f00 x1874672572683392/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830446 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830421.890988:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c68f00 x1874672572683392/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830446 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830421.890997:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830421.891001:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787830421.891005:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c68f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572683392:192.168.203.129@tcp:38:kworker.0 00000100:00100000:1.0:1787830421.891012:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c69a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572683520:192.168.203.129@tcp:38:kworker.0 00000800:00000100:1.0:1787830421.891035:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787830421.891048:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830421/real 1787830421] req@ffff8cb999c69a40 x1874672572683520/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830446 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830421.891059:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c69a40 x1874672572683520/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830446 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830421.891077:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830421.891081:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787830421.891085:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c69a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572683520:192.168.203.129@tcp:38:kworker.0 00010000:00010000:1.0:1787830422.722763:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787830423.746482:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787830424.770477:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787830425.794517:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787830426.819707:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787830427.011326:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787830427.011335:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787830427.011339:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787830427.011348:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 3379 00000100:00080000:1.0:1787830427.011358:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787830427.011365:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:1.0:1787830427.011396:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f392c0 x1874672572683648/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787830427.011425:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787830427.011428:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787830427.011430:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787830427.011434:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 3379 00000100:00080000:1.0:1787830427.011438:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787830427.011441:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:1.0:1787830427.011450:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787830427.011462:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f3a580 x1874672572683776/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787830427.011498:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787830427.011502:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1787830427.011504:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787830427.011508:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 3379 00000100:00080000:1.0:1787830427.011512:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787830427.011516:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:1.0:1787830427.011521:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787830427.011532:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f383c0 x1874672572683904/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787830427.011602:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f392c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572683648:192.168.203.129@tcp:250:kworker.0 00000800:00000100:2.0:1787830427.011657:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787830427.011702:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830427/real 1787830427] req@ffff8cb999f392c0 x1874672572683648/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830452 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787830427.011719:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f392c0 x1874672572683648/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830452 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787830427.011726:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787830427.011728:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787830427.011731:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f392c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572683648:192.168.203.129@tcp:250:kworker.0 00000100:00100000:2.0:1787830427.011736:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f3a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572683776:192.168.203.129@tcp:38:kworker.0 00000800:00000100:2.0:1787830427.011753:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787830427.011760:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830427/real 1787830427] req@ffff8cb999f3a580 x1874672572683776/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830452 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787830427.011771:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f3a580 x1874672572683776/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830452 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787830427.011779:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787830427.011782:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787830427.011785:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f3a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572683776:192.168.203.129@tcp:38:kworker.0 00000100:00100000:2.0:1787830427.011791:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f383c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572683904:192.168.203.129@tcp:38:kworker.0 00000800:00000100:2.0:1787830427.011817:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787830427.011827:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830427/real 1787830427] req@ffff8cb999f383c0 x1874672572683904/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830452 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787830427.011834:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f383c0 x1874672572683904/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830452 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787830427.011839:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787830427.011841:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787830427.011843:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f383c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572683904:192.168.203.129@tcp:38:kworker.0 00010000:00010000:1.0:1787830427.842640:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787830428.866517:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787830429.890454:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787830430.914468:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787830431.938417:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787830432.130424:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787830432.130435:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787830432.130439:0:37:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787830432.130448:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 3384 00000100:00080000:3.0:1787830432.130458:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787830432.130465:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:3.0:1787830432.130503:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb992316940 x1874672572684032/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787830432.130539:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787830432.130543:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787830432.130544:0:37:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787830432.130547:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 3384 00000100:00080000:3.0:1787830432.130551:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787830432.130554:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:3.0:1787830432.130563:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787830432.130573:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb992315a40 x1874672572684160/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787830432.130581:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787830432.130597:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1787830432.130599:0:37:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787830432.130602:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 3384 00000100:00080000:3.0:1787830432.130606:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787830432.130609:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:3.0:1787830432.130614:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787830432.130624:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb9923170c0 x1874672572684288/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787830432.130703:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb992316940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572684032:192.168.203.129@tcp:250:kworker.0 00000800:00000100:2.0:1787830432.130765:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787830432.130840:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830432/real 1787830432] req@ffff8cb992316940 x1874672572684032/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830457 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787830432.130857:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb992316940 x1874672572684032/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830457 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787830432.130867:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787830432.130871:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787830432.130874:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb992316940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572684032:192.168.203.129@tcp:250:kworker.0 00000100:00100000:2.0:1787830432.130881:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb992315a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572684160:192.168.203.129@tcp:38:kworker.0 00000800:00000100:2.0:1787830432.130903:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787830432.130916:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830432/real 1787830432] req@ffff8cb992315a40 x1874672572684160/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830457 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787830432.130959:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb992315a40 x1874672572684160/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830457 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787830432.130967:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787830432.130970:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787830432.130973:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb992315a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572684160:192.168.203.129@tcp:38:kworker.0 00000100:00100000:2.0:1787830432.130979:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb9923170c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572684288:192.168.203.129@tcp:38:kworker.0 00000800:00000100:2.0:1787830432.130998:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787830432.131008:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830432/real 1787830432] req@ffff8cb9923170c0 x1874672572684288/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830457 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787830432.131019:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb9923170c0 x1874672572684288/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830457 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787830432.131027:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787830432.131030:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787830432.131033:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb9923170c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572684288:192.168.203.129@tcp:38:kworker.0 00010000:00010000:0.0:1787830432.962680:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787830433.986746:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787830435.010933:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787830436.034778:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787830437.060035:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787830437.250491:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787830437.250501:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787830437.250506:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787830437.250516:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 3389 00000100:00080000:1.0:1787830437.250529:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787830437.250534:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:1.0:1787830437.250577:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f38f00 x1874672572684416/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787830437.250600:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787830437.250634:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787830437.250635:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787830437.250639:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 3389 00000100:00080000:1.0:1787830437.250645:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787830437.250648:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:1.0:1787830437.250659:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787830437.250671:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f38b40 x1874672572684544/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787830437.250681:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787830437.250684:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1787830437.250686:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1787830437.250686:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f38f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572684416:192.168.203.129@tcp:250:kworker.0 00000100:00080000:1.0:1787830437.250697:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 3389 00000100:00080000:1.0:1787830437.250704:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787830437.250708:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:1.0:1787830437.250713:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787830437.250727:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f3b0c0 x1874672572684672/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1787830437.250789:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787830437.250850:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830437/real 1787830437] req@ffff8cb999f38f00 x1874672572684416/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830462 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787830437.250870:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f38f00 x1874672572684416/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830462 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787830437.250902:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787830437.250908:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787830437.250912:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f38f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572684416:192.168.203.129@tcp:250:kworker.0 00000100:00100000:2.0:1787830437.250941:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f38b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572684544:192.168.203.129@tcp:38:kworker.0 00000800:00000100:2.0:1787830437.250965:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787830437.250980:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830437/real 1787830437] req@ffff8cb999f38b40 x1874672572684544/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830462 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787830437.250992:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f38b40 x1874672572684544/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830462 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787830437.251001:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787830437.251004:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787830437.251008:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f38b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572684544:192.168.203.129@tcp:38:kworker.0 00000100:00100000:2.0:1787830437.251015:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f3b0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572684672:192.168.203.129@tcp:38:kworker.0 00000800:00000100:2.0:1787830437.251035:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787830437.251043:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830437/real 1787830437] req@ffff8cb999f3b0c0 x1874672572684672/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830462 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787830437.251055:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f3b0c0 x1874672572684672/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830462 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787830437.251063:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787830437.251067:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787830437.251070:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f3b0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572684672:192.168.203.129@tcp:38:kworker.0 00010000:00010000:1.0:1787830438.084702:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787830439.106647:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787830440.130726:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787830441.154618:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787830442.178645:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787830442.370407:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787830442.370419:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787830442.370424:0:37:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787830442.370435:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 3394 00000100:00080000:3.0:1787830442.370447:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787830442.370455:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:3.0:1787830442.370494:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb9923161c0 x1874672572684800/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787830442.370520:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787830442.370523:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787830442.370525:0:37:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787830442.370528:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 3394 00000100:00080000:3.0:1787830442.370531:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787830442.370536:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:3.0:1787830442.370545:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787830442.370561:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb9923170c0 x1874672572684928/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787830442.370562:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb9923161c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572684800:192.168.203.129@tcp:250:kworker.0 00000100:00080000:3.0:1787830442.370596:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787830442.370600:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1787830442.370602:0:37:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787830442.370607:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 3394 00000100:00080000:3.0:1787830442.370624:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787830442.370630:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:3.0:1787830442.370636:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787830442.370649:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb992315a40 x1874672572685056/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787830442.370694:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb9923170c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572684928:192.168.203.129@tcp:38:kworker.0 00000100:00100000:2.0:1787830442.370726:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb992315a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572685056:192.168.203.129@tcp:38:kworker.0 00010000:00010000:1.0:1787830443.202723:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787830444.226462:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787830445.250416:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:3.0:1787830445.442703:0:2383:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.129:988 00000400:00000100:3.0:1787830445.442744:0:2383:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.129@tcp at host 192.168.203.129 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:3.0:1787830445.442761:0:2383:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.129@tcp 00000800:00000100:3.0:1787830445.442800:0:2383:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.129@tcp 00000800:00000100:3.0:1787830445.442808:0:2383:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.129@tcp 00000100:00100000:2.0:1787830445.443584:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830442/real 1787830445] req@ffff8cb9923170c0 x1874672572684928/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830467 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787830445.443614:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb9923170c0 x1874672572684928/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830467 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787830445.443626:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787830445.443630:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787830445.443634:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb9923170c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572684928:192.168.203.129@tcp:38:kworker.0 00000100:00100000:2.0:1787830445.443642:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830442/real 1787830445] req@ffff8cb992315a40 x1874672572685056/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830467 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787830445.443654:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb992315a40 x1874672572685056/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830467 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787830445.443661:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787830445.443664:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787830445.443667:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb992315a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572685056:192.168.203.129@tcp:38:kworker.0 00000100:00100000:2.0:1787830445.443674:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830442/real 1787830445] req@ffff8cb9923161c0 x1874672572684800/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830467 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787830445.443684:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb9923161c0 x1874672572684800/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830467 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787830445.443710:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787830445.443714:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787830445.443718:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb9923161c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572684800:192.168.203.129@tcp:250:kworker.0 00010000:00010000:1.0:1787830446.274631:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787830447.298428:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787830447.490597:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787830447.490607:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787830447.490610:0:37:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787830447.490619:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 3399 00000100:00080000:0.0:1787830447.490629:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787830447.490636:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:0.0:1787830447.490667:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c6bc00 x1874672572685184/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787830447.490697:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787830447.490701:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787830447.490703:0:37:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787830447.490706:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 3399 00000100:00080000:0.0:1787830447.490710:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787830447.490713:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:0.0:1787830447.490722:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787830447.490734:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c6b480 x1874672572685312/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787830447.490743:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787830447.490746:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1787830447.490747:0:37:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787830447.490750:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 3399 00000100:00080000:0.0:1787830447.490766:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787830447.490771:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:0.0:1787830447.490776:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787830447.490788:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c69e00 x1874672572685440/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787830447.490873:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c6bc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572685184:192.168.203.129@tcp:250:kworker.0 00000800:00000100:2.0:1787830447.490932:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787830447.490968:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830447/real 1787830447] req@ffff8cb999c6bc00 x1874672572685184/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830472 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787830447.490984:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c6bc00 x1874672572685184/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830472 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787830447.490995:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787830447.490999:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787830447.491003:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c6bc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572685184:192.168.203.129@tcp:250:kworker.0 00000100:00100000:2.0:1787830447.491009:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c6b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572685312:192.168.203.129@tcp:38:kworker.0 00000800:00000100:2.0:1787830447.491030:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787830447.491042:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830447/real 1787830447] req@ffff8cb999c6b480 x1874672572685312/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830472 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787830447.491052:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c6b480 x1874672572685312/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830472 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787830447.491060:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787830447.491063:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787830447.491067:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c6b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572685312:192.168.203.129@tcp:38:kworker.0 00000100:00100000:2.0:1787830447.491079:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c69e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572685440:192.168.203.129@tcp:38:kworker.0 00000800:00000100:2.0:1787830447.491100:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787830447.491111:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830447/real 1787830447] req@ffff8cb999c69e00 x1874672572685440/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830472 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787830447.491123:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c69e00 x1874672572685440/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830472 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787830447.491131:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787830447.491134:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787830447.491137:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c69e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572685440:192.168.203.129@tcp:38:kworker.0 00010000:00010000:0.0:1787830448.323411:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787830449.346589:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787830450.371479:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787830451.396570:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787830452.418472:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787830452.611824:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787830452.611834:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787830452.611839:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787830452.611848:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 3404 00000100:00080000:0.0:1787830452.611858:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787830452.611866:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:0.0:1787830452.611904:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c692c0 x1874672572685568/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787830452.611930:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787830452.611934:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787830452.611936:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787830452.611939:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 3404 00000100:00080000:0.0:1787830452.611943:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787830452.611946:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:2.0:1787830452.611982:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c692c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572685568:192.168.203.129@tcp:250:kworker.0 00000800:00000100:2.0:1787830452.612042:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787830452.612083:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830452/real 1787830452] req@ffff8cb999c692c0 x1874672572685568/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830477 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787830452.612099:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c692c0 x1874672572685568/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830477 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787830452.612110:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787830452.612113:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787830452.612118:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c692c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572685568:192.168.203.129@tcp:250:kworker.0 00000008:00100000:0.0:1787830452.612375:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787830452.612392:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c69680 x1874672572685696/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787830452.612433:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787830452.612437:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1787830452.612439:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787830452.612443:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 3404 00000100:00080000:0.0:1787830452.612448:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787830452.612452:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:0.0:1787830452.612457:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787830452.612470:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c6a1c0 x1874672572685824/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787830452.612560:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c69680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572685696:192.168.203.129@tcp:38:kworker.0 00000800:00000100:2.0:1787830452.612588:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787830452.612601:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830452/real 1787830452] req@ffff8cb999c69680 x1874672572685696/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830477 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787830452.612623:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c69680 x1874672572685696/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830477 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787830452.612632:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787830452.612635:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787830452.612638:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c69680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572685696:192.168.203.129@tcp:38:kworker.0 00000100:00100000:2.0:1787830452.612644:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c6a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572685824:192.168.203.129@tcp:38:kworker.0 00000800:00000100:2.0:1787830452.612663:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787830452.612673:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830452/real 1787830452] req@ffff8cb999c6a1c0 x1874672572685824/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830477 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787830452.612683:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c6a1c0 x1874672572685824/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830477 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787830452.612691:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787830452.612694:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787830452.612696:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c6a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572685824:192.168.203.129@tcp:38:kworker.0 00010000:00010000:1.0:1787830453.442422:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787830454.466451:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787830455.490449:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787830456.514457:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787830457.538565:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787830457.731315:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787830457.731325:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787830457.731330:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787830457.731339:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 3409 00000100:00080000:0.0:1787830457.731350:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787830457.731357:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:0.0:1787830457.731389:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c68000 x1874672572685952/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787830457.731423:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787830457.731426:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787830457.731428:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787830457.731432:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 3409 00000100:00080000:0.0:1787830457.731448:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787830457.731453:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:0.0:1787830457.731462:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787830457.731485:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c6a580 x1874672572686080/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787830457.731495:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787830457.731498:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1787830457.731500:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787830457.731504:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 3409 00000100:00080000:0.0:1787830457.731508:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787830457.731511:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:0.0:1787830457.731515:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787830457.731527:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c6a1c0 x1874672572686208/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787830457.731615:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c68000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572685952:192.168.203.129@tcp:250:kworker.0 00000800:00000100:2.0:1787830457.731676:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787830457.731718:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830457/real 1787830457] req@ffff8cb999c68000 x1874672572685952/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830482 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787830457.731734:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c68000 x1874672572685952/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830482 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787830457.731756:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787830457.731760:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787830457.731764:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c68000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572685952:192.168.203.129@tcp:250:kworker.0 00000100:00100000:2.0:1787830457.731771:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c6a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572686080:192.168.203.129@tcp:38:kworker.0 00000800:00000100:2.0:1787830457.731795:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787830457.731807:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830457/real 1787830457] req@ffff8cb999c6a580 x1874672572686080/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830482 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787830457.731819:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c6a580 x1874672572686080/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830482 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787830457.731826:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787830457.731829:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787830457.731832:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c6a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572686080:192.168.203.129@tcp:38:kworker.0 00000100:00100000:2.0:1787830457.731838:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c6a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572686208:192.168.203.129@tcp:38:kworker.0 00000800:00000100:2.0:1787830457.731860:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787830457.731871:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830457/real 1787830457] req@ffff8cb999c6a1c0 x1874672572686208/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830482 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787830457.731880:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c6a1c0 x1874672572686208/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830482 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787830457.731887:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787830457.731890:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787830457.731894:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c6a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572686208:192.168.203.129@tcp:38:kworker.0 00010000:00010000:1.0:1787830458.562992:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787830459.586440:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787830460.610719:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787830461.634482:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787830462.658417:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787830462.850552:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787830462.850561:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787830462.850565:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787830462.850576:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 3415 00000100:00080000:1.0:1787830462.850587:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787830462.850594:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:1.0:1787830462.850629:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f3b480 x1874672572686336/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787830462.850658:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787830462.850663:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787830462.850665:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787830462.850669:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 3415 00000100:00080000:1.0:1787830462.850673:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787830462.850677:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:1.0:1787830462.850686:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787830462.850692:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f3b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572686336:192.168.203.129@tcp:250:kworker.0 00000100:00100000:1.0:1787830462.850701:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f3b840 x1874672572686464/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787830462.850731:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787830462.850735:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1787830462.850737:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787830462.850741:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 3415 00000100:00080000:1.0:1787830462.850747:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787830462.850751:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000800:00000100:3.0:1787830462.850756:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000008:00100000:1.0:1787830462.850757:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787830462.850773:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f39a40 x1874672572686592/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1787830462.850795:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830462/real 1787830462] req@ffff8cb999f3b480 x1874672572686336/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830487 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787830462.850808:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f3b480 x1874672572686336/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830487 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787830462.850816:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787830462.850819:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:3.0:1787830462.850827:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f3b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572686336:192.168.203.129@tcp:250:kworker.0 00000100:00100000:3.0:1787830462.850845:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f3b840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572686464:192.168.203.129@tcp:38:kworker.0 00000800:00000100:3.0:1787830462.850864:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:3.0:1787830462.850872:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830462/real 1787830462] req@ffff8cb999f3b840 x1874672572686464/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830487 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787830462.850881:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f3b840 x1874672572686464/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830487 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787830462.850887:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787830462.850890:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:3.0:1787830462.850893:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f3b840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572686464:192.168.203.129@tcp:38:kworker.0 00000100:00100000:3.0:1787830462.850897:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f39a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572686592:192.168.203.129@tcp:38:kworker.0 00000800:00000100:3.0:1787830462.850910:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:3.0:1787830462.850916:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830462/real 1787830462] req@ffff8cb999f39a40 x1874672572686592/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830487 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787830462.850923:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f39a40 x1874672572686592/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830487 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787830462.850928:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787830462.850930:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:3.0:1787830462.850932:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f39a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572686592:192.168.203.129@tcp:38:kworker.0 00010000:00010000:0.0:1787830463.682528:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787830464.706448:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787830465.731448:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787830466.755021:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787830467.779064:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787830467.970432:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787830467.970443:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787830467.970447:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787830467.970457:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 3420 00000100:00080000:1.0:1787830467.970467:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787830467.970475:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:1.0:1787830467.970509:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f39e00 x1874672572686720/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787830467.970539:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787830467.970544:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787830467.970546:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787830467.970565:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 3420 00000100:00080000:1.0:1787830467.970569:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787830467.970573:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:1.0:1787830467.970583:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787830467.970597:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f39a40 x1874672572686848/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787830467.970606:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787830467.970609:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1787830467.970611:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787830467.970614:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 3420 00000100:00080000:1.0:1787830467.970617:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787830467.970621:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:1.0:1787830467.970625:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787830467.970636:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f3b840 x1874672572686976/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1787830467.970789:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f39e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572686720:192.168.203.129@tcp:250:kworker.0 00000800:00000100:3.0:1787830467.970838:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:3.0:1787830467.970881:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830467/real 1787830467] req@ffff8cb999f39e00 x1874672572686720/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830492 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787830467.970894:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f39e00 x1874672572686720/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830492 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787830467.970901:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787830467.970904:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:3.0:1787830467.970906:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f39e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572686720:192.168.203.129@tcp:250:kworker.0 00000100:00100000:3.0:1787830467.970911:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f39a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572686848:192.168.203.129@tcp:38:kworker.0 00000800:00000100:3.0:1787830467.970925:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:3.0:1787830467.970933:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830467/real 1787830467] req@ffff8cb999f39a40 x1874672572686848/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830492 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787830467.970940:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f39a40 x1874672572686848/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830492 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787830467.970946:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787830467.970948:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:3.0:1787830467.970949:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f39a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572686848:192.168.203.129@tcp:38:kworker.0 00000100:00100000:3.0:1787830467.970952:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f3b840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572686976:192.168.203.129@tcp:38:kworker.0 00000800:00000100:3.0:1787830467.970961:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:3.0:1787830467.970966:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830467/real 1787830467] req@ffff8cb999f3b840 x1874672572686976/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830492 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787830467.970972:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f3b840 x1874672572686976/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830492 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787830467.970985:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787830467.970987:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:3.0:1787830467.970989:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f3b840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572686976:192.168.203.129@tcp:38:kworker.0 00010000:00010000:1.0:1787830468.802441:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787830469.826399:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787830470.851480:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787830471.874471:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787830472.898505:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787830473.090610:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787830473.090620:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787830473.090623:0:37:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787830473.090631:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 3425 00000100:00080000:3.0:1787830473.090640:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787830473.090647:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:3.0:1787830473.090736:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb992316940 x1874672572687104/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787830473.090975:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb992316940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572687104:192.168.203.129@tcp:250:kworker.0 00000800:00000100:2.0:1787830473.091033:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00080000:3.0:1787830473.091062:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787830473.091066:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787830473.091069:0:37:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787830473.091073:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 3425 00000100:00080000:3.0:1787830473.091077:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787830473.091081:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:3.0:1787830473.091091:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787830473.091104:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb992315e00 x1874672572687232/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787830473.092540:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830473/real 1787830473] req@ffff8cb992316940 x1874672572687104/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830498 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787830473.092564:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb992316940 x1874672572687104/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830498 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787830473.092575:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787830473.092579:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787830473.092583:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb992316940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572687104:192.168.203.129@tcp:250:kworker.0 00000100:00100000:2.0:1787830473.092609:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb992315e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572687232:192.168.203.129@tcp:38:kworker.0 00000800:00000100:2.0:1787830473.092649:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787830473.092663:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830473/real 1787830473] req@ffff8cb992315e00 x1874672572687232/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830498 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787830473.092680:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb992315e00 x1874672572687232/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830498 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787830473.092687:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787830473.092701:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787830473.092705:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb992315e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572687232:192.168.203.129@tcp:38:kworker.0 00000100:00080000:3.0:1787830473.093795:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787830473.093802:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1787830473.093819:0:37:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787830473.093825:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 3425 00000100:00080000:3.0:1787830473.093832:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787830473.093838:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:3.0:1787830473.093859:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787830473.093884:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb992314f00 x1874672572687360/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787830473.093937:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb992314f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572687360:192.168.203.129@tcp:38:kworker.0 00000800:00000100:2.0:1787830473.093986:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787830473.094447:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830473/real 1787830473] req@ffff8cb992314f00 x1874672572687360/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830498 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787830473.094474:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb992314f00 x1874672572687360/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830498 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787830473.094484:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787830473.094488:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787830473.094492:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb992314f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572687360:192.168.203.129@tcp:38:kworker.0 00010000:00010000:1.0:1787830473.922836:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787830474.946521:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787830475.970773:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787830476.994913:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787830478.018625:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787830478.210421:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787830478.210431:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787830478.210435:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787830478.210445:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 3430 00000100:00080000:0.0:1787830478.210455:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787830478.210479:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:0.0:1787830478.210511:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c69680 x1874672572687488/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787830478.210542:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787830478.210546:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787830478.210547:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787830478.210551:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 3430 00000100:00080000:0.0:1787830478.210555:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787830478.210559:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:0.0:1787830478.210569:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787830478.210581:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c692c0 x1874672572687616/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787830478.210590:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787830478.210593:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1787830478.210595:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787830478.210598:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 3430 00000100:00080000:0.0:1787830478.210601:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787830478.210604:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:0.0:1787830478.210608:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787830478.210619:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c69e00 x1874672572687744/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787830478.210738:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c69680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572687488:192.168.203.129@tcp:250:kworker.0 00000800:00000100:2.0:1787830478.210801:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787830478.210840:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830478/real 1787830478] req@ffff8cb999c69680 x1874672572687488/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830503 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787830478.210856:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c69680 x1874672572687488/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830503 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787830478.210866:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787830478.210869:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787830478.210873:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c69680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572687488:192.168.203.129@tcp:250:kworker.0 00000100:00100000:2.0:1787830478.210880:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c692c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572687616:192.168.203.129@tcp:38:kworker.0 00000800:00000100:2.0:1787830478.210902:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787830478.210915:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830478/real 1787830478] req@ffff8cb999c692c0 x1874672572687616/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830503 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787830478.210927:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c692c0 x1874672572687616/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830503 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787830478.210948:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787830478.210952:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787830478.210956:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c692c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572687616:192.168.203.129@tcp:38:kworker.0 00000100:00100000:2.0:1787830478.210961:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c69e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572687744:192.168.203.129@tcp:38:kworker.0 00000800:00000100:2.0:1787830478.210981:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787830478.210992:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830478/real 1787830478] req@ffff8cb999c69e00 x1874672572687744/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830503 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787830478.211003:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c69e00 x1874672572687744/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830503 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787830478.211010:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787830478.211013:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787830478.211016:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c69e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572687744:192.168.203.129@tcp:38:kworker.0 00010000:00010000:0.0:1787830479.042549:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787830480.068933:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787830481.090463:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787830482.114417:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787830483.138479:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787830483.330568:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787830483.330603:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787830483.330608:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787830483.330620:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 3435 00000100:00080000:1.0:1787830483.330633:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787830483.330641:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:1.0:1787830483.330681:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f3b480 x1874672572687872/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787830483.330719:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787830483.330724:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787830483.330726:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787830483.330730:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 3435 00000100:00080000:1.0:1787830483.330734:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787830483.330738:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:1.0:1787830483.330750:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787830483.330762:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f3b0c0 x1874672572688000/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787830483.330772:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787830483.330776:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1787830483.331873:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787830483.331882:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 3435 00000100:00080000:1.0:1787830483.331889:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787830483.331895:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:2.0:1787830483.331899:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f3b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572687872:192.168.203.129@tcp:250:kworker.0 00000008:00100000:1.0:1787830483.331904:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787830483.331923:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f38b40 x1874672572688128/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1787830483.331981:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787830483.332029:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830483/real 1787830483] req@ffff8cb999f3b480 x1874672572687872/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830508 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787830483.332048:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f3b480 x1874672572687872/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830508 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787830483.332059:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787830483.332063:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787830483.332073:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f3b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572687872:192.168.203.129@tcp:250:kworker.0 00000100:00100000:2.0:1787830483.332081:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f3b0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572688000:192.168.203.129@tcp:38:kworker.0 00000800:00000100:2.0:1787830483.332106:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787830483.332119:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830483/real 1787830483] req@ffff8cb999f3b0c0 x1874672572688000/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830508 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787830483.332131:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f3b0c0 x1874672572688000/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830508 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787830483.332140:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787830483.332143:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787830483.332147:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f3b0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572688000:192.168.203.129@tcp:38:kworker.0 00000100:00100000:2.0:1787830483.332173:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f38b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572688128:192.168.203.129@tcp:38:kworker.0 00000800:00000100:2.0:1787830483.332193:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787830483.332203:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830483/real 1787830483] req@ffff8cb999f38b40 x1874672572688128/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830508 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787830483.332215:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f38b40 x1874672572688128/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830508 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787830483.332223:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787830483.332226:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787830483.332230:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f38b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572688128:192.168.203.129@tcp:38:kworker.0 00010000:00010000:1.0:1787830484.162685:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787830485.187339:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787830486.210458:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787830487.234400:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787830488.259091:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787830488.450416:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787830488.450429:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787830488.450434:0:37:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787830488.450447:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 3440 00000100:00080000:1.0:1787830488.450461:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787830488.450470:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:1.0:1787830488.450513:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f38f00 x1874672572688256/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787830488.450547:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787830488.450552:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787830488.450554:0:37:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787830488.450561:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 3440 00000100:00080000:1.0:1787830488.450565:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787830488.450570:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:1.0:1787830488.450582:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787830488.450590:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f38f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572688256:192.168.203.129@tcp:250:kworker.0 00000100:00100000:1.0:1787830488.450597:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f383c0 x1874672572688384/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787830488.450608:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787830488.450613:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1787830488.450614:0:37:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787830488.450637:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 3440 00000100:00080000:1.0:1787830488.450642:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787830488.450646:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:1.0:1787830488.450651:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787830488.450663:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f3a580 x1874672572688512/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1787830488.450743:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787830488.450806:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830488/real 1787830488] req@ffff8cb999f38f00 x1874672572688256/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830513 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787830488.450826:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f38f00 x1874672572688256/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830513 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787830488.450838:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787830488.450842:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787830488.450846:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f38f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572688256:192.168.203.129@tcp:250:kworker.0 00000100:00100000:2.0:1787830488.450874:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f383c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572688384:192.168.203.129@tcp:38:kworker.0 00000800:00000100:2.0:1787830488.450899:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787830488.450913:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830488/real 1787830488] req@ffff8cb999f383c0 x1874672572688384/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830513 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787830488.450926:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f383c0 x1874672572688384/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830513 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787830488.451024:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787830488.451029:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787830488.451033:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f383c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572688384:192.168.203.129@tcp:38:kworker.0 00000100:00100000:2.0:1787830488.451039:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f3a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572688512:192.168.203.129@tcp:38:kworker.0 00000800:00000100:2.0:1787830488.451063:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787830488.451075:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830488/real 1787830488] req@ffff8cb999f3a580 x1874672572688512/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830513 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787830488.451088:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f3a580 x1874672572688512/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830513 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787830488.451108:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787830488.451111:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787830488.451115:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f3a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572688512:192.168.203.129@tcp:38:kworker.0 00010000:00010000:1.0:1787830489.282484:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787830490.306928:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787830491.331719:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787830492.355086:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787830493.378597:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787830493.571281:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787830493.571290:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787830493.571294:0:37:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787830493.571303:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 3445 00000100:00080000:3.0:1787830493.571313:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787830493.571320:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:3.0:1787830493.571354:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb992314780 x1874672572688640/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787830493.571380:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787830493.571385:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787830493.571386:0:37:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787830493.571390:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 3445 00000100:00080000:3.0:1787830493.571393:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787830493.571396:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:3.0:1787830493.571404:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787830493.571416:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb992314b40 x1874672572688768/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787830493.571453:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787830493.571457:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1787830493.571459:0:37:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787830493.571463:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 3445 00000100:00080000:3.0:1787830493.571466:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787830493.571470:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:3.0:1787830493.571475:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787830493.571486:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb992317480 x1874672572688896/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787830493.571922:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb992314780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572688640:192.168.203.129@tcp:250:kworker.0 00000800:00000100:2.0:1787830493.573241:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787830493.573318:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830493/real 1787830493] req@ffff8cb992314780 x1874672572688640/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830518 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787830493.573336:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb992314780 x1874672572688640/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830518 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787830493.573347:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787830493.573351:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787830493.573355:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb992314780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572688640:192.168.203.129@tcp:250:kworker.0 00000100:00100000:2.0:1787830493.573363:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb992314b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572688768:192.168.203.129@tcp:38:kworker.0 00000800:00000100:2.0:1787830493.573397:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787830493.573410:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830493/real 1787830493] req@ffff8cb992314b40 x1874672572688768/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830518 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787830493.573422:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb992314b40 x1874672572688768/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830518 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787830493.573430:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787830493.573434:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787830493.573437:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb992314b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572688768:192.168.203.129@tcp:38:kworker.0 00000100:00100000:2.0:1787830493.573453:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb992317480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572688896:192.168.203.129@tcp:38:kworker.0 00000800:00000100:2.0:1787830493.573475:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787830493.573486:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830493/real 1787830493] req@ffff8cb992317480 x1874672572688896/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830518 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787830493.573498:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb992317480 x1874672572688896/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830518 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787830493.573504:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787830493.573508:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787830493.573512:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb992317480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572688896:192.168.203.129@tcp:38:kworker.0 00010000:00010000:0.0:1787830494.403707:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787830495.427252:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787830496.450758:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787830497.474395:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787830498.498495:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787830498.690404:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787830498.690414:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787830498.690418:0:37:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787830498.690428:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 3450 00000100:00080000:3.0:1787830498.690442:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787830498.690451:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:3.0:1787830498.690498:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb992316580 x1874672572689024/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787830498.690532:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787830498.690536:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787830498.690538:0:37:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787830498.690542:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 3450 00000100:00080000:3.0:1787830498.690546:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787830498.690549:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:3.0:1787830498.690573:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787830498.690587:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb992317480 x1874672572689152/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787830498.690598:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787830498.690601:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1787830498.690603:0:37:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787830498.690606:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 3450 00000100:00080000:3.0:1787830498.690610:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787830498.690614:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:3.0:1787830498.690619:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787830498.690629:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb992314b40 x1874672572689280/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787830498.691592:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb992316580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572689024:192.168.203.129@tcp:250:kworker.0 00000800:00000100:2.0:1787830498.691656:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787830498.691709:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830498/real 1787830498] req@ffff8cb992316580 x1874672572689024/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830523 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787830498.691727:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb992316580 x1874672572689024/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830523 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787830498.691738:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787830498.691743:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787830498.691747:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb992316580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572689024:192.168.203.129@tcp:250:kworker.0 00000100:00100000:2.0:1787830498.691754:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb992317480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572689152:192.168.203.129@tcp:38:kworker.0 00000800:00000100:2.0:1787830498.691788:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787830498.691802:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830498/real 1787830498] req@ffff8cb992317480 x1874672572689152/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830523 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787830498.691980:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb992317480 x1874672572689152/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830523 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787830498.691989:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787830498.691992:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787830498.691996:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb992317480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572689152:192.168.203.129@tcp:38:kworker.0 00000100:00100000:2.0:1787830498.692002:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb992314b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572689280:192.168.203.129@tcp:38:kworker.0 00000800:00000100:2.0:1787830498.692027:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787830498.692039:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830498/real 1787830498] req@ffff8cb992314b40 x1874672572689280/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830523 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787830498.692051:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb992314b40 x1874672572689280/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830523 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787830498.692059:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787830498.692062:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787830498.692065:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb992314b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572689280:192.168.203.129@tcp:38:kworker.0 00010000:00010000:1.0:1787830499.522543:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787830500.559311:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787830501.573085:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787830502.597469:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787830503.619215:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787830503.810553:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787830503.810564:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787830503.810569:0:37:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787830503.810578:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 3455 00000100:00080000:1.0:1787830503.810601:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787830503.810608:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:1.0:1787830503.810645:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f392c0 x1874672572689408/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787830503.810684:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787830503.810688:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787830503.810704:0:37:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787830503.810709:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 3455 00000100:00080000:1.0:1787830503.810713:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787830503.810717:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:1.0:1787830503.810727:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787830503.810741:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f3a580 x1874672572689536/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787830503.810752:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787830503.810756:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1787830503.810758:0:37:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787830503.810761:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 3455 00000100:00080000:1.0:1787830503.810764:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787830503.810768:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:1.0:1787830503.810773:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787830503.810795:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f383c0 x1874672572689664/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787830503.812124:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f392c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572689408:192.168.203.129@tcp:250:kworker.0 00000800:00000100:2.0:1787830503.812205:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787830503.812325:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830503/real 1787830503] req@ffff8cb999f392c0 x1874672572689408/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830528 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787830503.812342:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f392c0 x1874672572689408/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830528 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787830503.812370:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787830503.812374:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787830503.812379:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f392c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572689408:192.168.203.129@tcp:250:kworker.0 00000100:00100000:2.0:1787830503.812386:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f3a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572689536:192.168.203.129@tcp:38:kworker.0 00000800:00000100:2.0:1787830503.812411:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787830503.812423:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830503/real 1787830503] req@ffff8cb999f3a580 x1874672572689536/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830528 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787830503.812435:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f3a580 x1874672572689536/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830528 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787830503.812442:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787830503.812446:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787830503.812449:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f3a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572689536:192.168.203.129@tcp:38:kworker.0 00000100:00100000:2.0:1787830503.812456:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f383c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572689664:192.168.203.129@tcp:38:kworker.0 00000800:00000100:2.0:1787830503.812475:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1787830503.812485:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830503/real 1787830503] req@ffff8cb999f383c0 x1874672572689664/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830528 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787830503.812497:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f383c0 x1874672572689664/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830528 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787830503.812505:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787830503.812508:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787830503.812512:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f383c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572689664:192.168.203.129@tcp:38:kworker.0 00010000:00010000:1.0:1787830504.643040:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787830505.666436:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787830506.690429:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787830507.716316:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787830508.738466:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787830508.934457:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787830508.934466:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787830508.934470:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787830508.934480:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 3461 00000100:00080000:2.0:1787830508.934491:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787830508.934498:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:2.0:1787830508.934532:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8e580 x1874672572689792/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787830508.934569:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787830508.934573:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787830508.934575:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787830508.934578:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 3461 00000100:00080000:2.0:1787830508.934582:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787830508.934586:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:2.0:1787830508.934595:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787830508.934606:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8d680 x1874672572689920/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787830508.934614:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787830508.934618:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1787830508.934619:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787830508.934623:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 3461 00000100:00080000:2.0:1787830508.934626:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787830508.934629:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:2.0:1787830508.934648:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787830508.934658:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8f480 x1874672572690048/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1787830508.934759:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8e580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572689792:192.168.203.129@tcp:250:kworker.0 00000100:00100000:3.0:1787830508.934886:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8d680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572689920:192.168.203.129@tcp:38:kworker.0 00000100:00100000:3.0:1787830508.934914:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572690048:192.168.203.129@tcp:38:kworker.0 00010000:00010000:0.0:1787830509.762871:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787830510.786917:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787830511.810700:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:3.0:1787830512.003178:0:2384:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.129:988 00000400:00000100:3.0:1787830512.003220:0:2384:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.129@tcp at host 192.168.203.129 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:3.0:1787830512.003235:0:2384:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.129@tcp 00000800:00000100:3.0:1787830512.003300:0:2384:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.129@tcp 00000800:00000100:3.0:1787830512.003310:0:2384:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.129@tcp 00000100:00100000:2.0:1787830512.004973:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830508/real 1787830512] req@ffff8cb998a8e580 x1874672572689792/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830533 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787830512.005006:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8e580 x1874672572689792/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830533 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787830512.005018:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787830512.005023:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787830512.005027:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8e580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572689792:192.168.203.129@tcp:250:kworker.0 00000100:00100000:2.0:1787830512.005035:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830508/real 1787830512] req@ffff8cb998a8d680 x1874672572689920/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830533 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787830512.005047:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8d680 x1874672572689920/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830533 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787830512.005054:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787830512.005057:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787830512.005061:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8d680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572689920:192.168.203.129@tcp:38:kworker.0 00000100:00100000:2.0:1787830512.005068:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830508/real 1787830512] req@ffff8cb998a8f480 x1874672572690048/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830533 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787830512.005080:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8f480 x1874672572690048/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830533 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787830512.005119:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787830512.005123:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:2.0:1787830512.005127:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572690048:192.168.203.129@tcp:38:kworker.0 00010000:00010000:0.0:1787830512.835359:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787830513.858574:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787830514.051177:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787830514.051187:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787830514.051192:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787830514.051203:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 3466 00000100:00080000:2.0:1787830514.051213:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787830514.051220:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:2.0:1787830514.051256:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8f480 x1874672572690176/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787830514.051313:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787830514.051317:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787830514.051319:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787830514.051323:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 3466 00000100:00080000:2.0:1787830514.051327:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787830514.051331:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:2.0:1787830514.051342:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787830514.051356:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8d680 x1874672572690304/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787830514.051365:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787830514.051368:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1787830514.051387:0:8:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787830514.051392:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 3466 00000100:00080000:2.0:1787830514.051397:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787830514.051402:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:2.0:1787830514.051409:0:8:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787830514.051424:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8e580 x1874672572690432/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787830514.051510:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572690176:192.168.203.129@tcp:250:kworker.0 00000800:00000100:0.0:1787830514.051571:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787830514.051611:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830514/real 1787830514] req@ffff8cb998a8f480 x1874672572690176/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830539 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830514.051628:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8f480 x1874672572690176/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830539 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830514.051639:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830514.051642:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787830514.051646:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572690176:192.168.203.129@tcp:250:kworker.0 00000100:00100000:0.0:1787830514.051653:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8d680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572690304:192.168.203.129@tcp:38:kworker.0 00000800:00000100:0.0:1787830514.051677:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787830514.051690:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830514/real 1787830514] req@ffff8cb998a8d680 x1874672572690304/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830539 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830514.051702:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8d680 x1874672572690304/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830539 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830514.051723:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830514.051726:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787830514.051730:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8d680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572690304:192.168.203.129@tcp:38:kworker.0 00000100:00100000:0.0:1787830514.051736:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8e580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572690432:192.168.203.129@tcp:38:kworker.0 00000800:00000100:0.0:1787830514.051761:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787830514.051773:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830514/real 1787830514] req@ffff8cb998a8e580 x1874672572690432/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830539 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830514.051785:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8e580 x1874672572690432/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830539 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830514.051794:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830514.051797:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787830514.051800:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8e580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572690432:192.168.203.129@tcp:38:kworker.0 00010000:00010000:1.0:1787830514.885498:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787830515.906478:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787830516.930682:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787830517.956137:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787830518.989405:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787830519.170396:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787830519.170406:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787830519.170410:0:37:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787830519.170417:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 3471 00000100:00080000:2.0:1787830519.170426:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787830519.170432:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:2.0:1787830519.170468:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8c000 x1874672572690560/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787830519.170503:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787830519.170507:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787830519.170509:0:37:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787830519.170513:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 3471 00000100:00080000:2.0:1787830519.170516:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787830519.170520:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:2.0:1787830519.170529:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787830519.170544:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8cb40 x1874672572690688/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787830519.170561:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572690560:192.168.203.129@tcp:250:kworker.0 00000100:00080000:2.0:1787830519.170579:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787830519.170583:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1787830519.170585:0:37:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787830519.170589:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 3471 00000100:00080000:2.0:1787830519.170593:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787830519.170597:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:2.0:1787830519.170603:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787830519.170614:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8ed00 x1874672572690816/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1787830519.170719:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787830519.170756:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830519/real 1787830519] req@ffff8cb998a8c000 x1874672572690560/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830544 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830519.170769:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8c000 x1874672572690560/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830544 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830519.170776:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830519.170784:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787830519.170787:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572690560:192.168.203.129@tcp:250:kworker.0 00000100:00100000:0.0:1787830519.170806:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8cb40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572690688:192.168.203.129@tcp:38:kworker.0 00000800:00000100:0.0:1787830519.170823:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787830519.170831:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830519/real 1787830519] req@ffff8cb998a8cb40 x1874672572690688/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830544 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830519.170838:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8cb40 x1874672572690688/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830544 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830519.170843:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830519.170845:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787830519.170848:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8cb40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572690688:192.168.203.129@tcp:38:kworker.0 00000100:00100000:0.0:1787830519.170851:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8ed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572690816:192.168.203.129@tcp:38:kworker.0 00000800:00000100:0.0:1787830519.170860:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787830519.170865:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830519/real 1787830519] req@ffff8cb998a8ed00 x1874672572690816/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830544 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830519.170871:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8ed00 x1874672572690816/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830544 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830519.170876:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830519.170877:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787830519.170879:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8ed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572690816:192.168.203.129@tcp:38:kworker.0 00010000:00010000:1.0:1787830520.002561:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787830521.027991:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787830522.051183:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787830523.074479:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787830524.098480:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787830524.291666:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787830524.291676:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787830524.291694:0:37:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787830524.291702:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 3476 00000100:00080000:3.0:1787830524.291712:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787830524.291718:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:3.0:1787830524.291748:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb992314780 x1874672572690944/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787830524.291823:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb992314780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572690944:192.168.203.129@tcp:250:kworker.0 00000800:00000100:0.0:1787830524.291877:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00080000:3.0:1787830524.291907:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787830524.291911:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787830524.291912:0:37:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787830524.291916:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 3476 00000100:00080000:3.0:1787830524.291920:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787830524.291924:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:3.0:1787830524.291932:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787830524.291943:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb992314f00 x1874672572691072/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787830524.292641:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830524/real 1787830524] req@ffff8cb992314780 x1874672572690944/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830549 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830524.292659:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb992314780 x1874672572690944/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830549 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830524.292669:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830524.292672:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787830524.292676:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb992314780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572690944:192.168.203.129@tcp:250:kworker.0 00000100:00100000:0.0:1787830524.292698:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb992314f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572691072:192.168.203.129@tcp:38:kworker.0 00000800:00000100:0.0:1787830524.292725:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787830524.292838:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830524/real 1787830524] req@ffff8cb992314f00 x1874672572691072/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830549 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830524.292851:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb992314f00 x1874672572691072/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830549 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830524.292859:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830524.292862:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787830524.292865:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb992314f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572691072:192.168.203.129@tcp:38:kworker.0 00000100:00080000:3.0:1787830524.293550:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787830524.293555:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1787830524.293557:0:37:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787830524.293621:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 3476 00000100:00080000:3.0:1787830524.293627:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787830524.293633:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:3.0:1787830524.293641:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787830524.293661:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb992315e00 x1874672572691200/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787830524.293786:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb992315e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572691200:192.168.203.129@tcp:38:kworker.0 00000800:00000100:0.0:1787830524.293820:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787830524.293894:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830524/real 1787830524] req@ffff8cb992315e00 x1874672572691200/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830549 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830524.293919:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb992315e00 x1874672572691200/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830549 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830524.293927:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830524.293930:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787830524.293934:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb992315e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572691200:192.168.203.129@tcp:38:kworker.0 00010000:00010000:0.0:1787830525.122455:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787830526.147068:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787830527.170466:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787830528.194993:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787830529.218631:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787830529.410583:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787830529.410595:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787830529.410599:0:37:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787830529.410609:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 3481 00000100:00080000:1.0:1787830529.410618:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787830529.410626:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:1.0:1787830529.410661:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f38f00 x1874672572691328/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787830529.410699:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787830529.410703:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787830529.410706:0:37:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787830529.410715:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 3481 00000100:00080000:1.0:1787830529.410718:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787830529.410722:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:1.0:1787830529.410731:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787830529.410742:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f38b40 x1874672572691456/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787830529.410752:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787830529.410755:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1787830529.410770:0:37:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787830529.410773:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 3481 00000100:00080000:1.0:1787830529.410777:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787830529.410781:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:1.0:1787830529.410786:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787830529.410797:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f3b0c0 x1874672572691584/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787830529.410947:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f38f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572691328:192.168.203.129@tcp:250:kworker.0 00000800:00000100:0.0:1787830529.411050:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787830529.411087:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830529/real 1787830529] req@ffff8cb999f38f00 x1874672572691328/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830554 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830529.411103:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f38f00 x1874672572691328/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830554 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830529.411114:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830529.411118:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787830529.411122:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f38f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572691328:192.168.203.129@tcp:250:kworker.0 00000100:00100000:0.0:1787830529.411129:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f38b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572691456:192.168.203.129@tcp:38:kworker.0 00000800:00000100:0.0:1787830529.411151:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787830529.411163:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830529/real 1787830529] req@ffff8cb999f38b40 x1874672572691456/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830554 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830529.411177:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f38b40 x1874672572691456/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830554 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830529.411191:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830529.411194:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787830529.411198:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f38b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572691456:192.168.203.129@tcp:38:kworker.0 00000100:00100000:0.0:1787830529.411204:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f3b0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572691584:192.168.203.129@tcp:38:kworker.0 00000800:00000100:0.0:1787830529.411226:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787830529.411238:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830529/real 1787830529] req@ffff8cb999f3b0c0 x1874672572691584/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830554 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830529.411249:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f3b0c0 x1874672572691584/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830554 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830529.411257:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830529.411285:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787830529.411289:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f3b0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572691584:192.168.203.129@tcp:38:kworker.0 00010000:00010000:1.0:1787830530.242707:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787830531.267340:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787830532.291497:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787830533.314581:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787830534.340521:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787830534.530925:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787830534.530947:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787830534.530952:0:37:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787830534.530965:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 3486 00000100:00080000:3.0:1787830534.530979:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787830534.530987:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:3.0:1787830534.531031:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb992316940 x1874672572691712/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787830534.531070:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787830534.531074:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787830534.531076:0:37:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787830534.531080:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 3486 00000100:00080000:3.0:1787830534.531162:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787830534.531167:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:3.0:1787830534.531178:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787830534.531192:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb9923161c0 x1874672572691840/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787830534.531203:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1787830534.531856:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb992316940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572691712:192.168.203.129@tcp:250:kworker.0 00000800:00000100:0.0:1787830534.531995:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787830534.532045:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830534/real 1787830534] req@ffff8cb992316940 x1874672572691712/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830559 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830534.532064:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb992316940 x1874672572691712/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830559 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830534.532074:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830534.532093:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787830534.532098:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb992316940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572691712:192.168.203.129@tcp:250:kworker.0 00000100:00100000:0.0:1787830534.532107:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb9923161c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572691840:192.168.203.129@tcp:38:kworker.0 00000800:00000100:0.0:1787830534.532131:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787830534.532144:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830534/real 1787830534] req@ffff8cb9923161c0 x1874672572691840/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830559 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830534.532155:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb9923161c0 x1874672572691840/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830559 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830534.532162:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830534.532165:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787830534.532168:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb9923161c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572691840:192.168.203.129@tcp:38:kworker.0 00000100:00080000:3.0:1787830534.533353:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1787830534.533359:0:37:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787830534.533369:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 3486 00000100:00080000:3.0:1787830534.533377:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787830534.533384:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:3.0:1787830534.533406:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787830534.533437:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb992315a40 x1874672572691968/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787830534.533786:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb992315a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572691968:192.168.203.129@tcp:38:kworker.0 00000800:00000100:0.0:1787830534.533833:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787830534.533854:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830534/real 1787830534] req@ffff8cb992315a40 x1874672572691968/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830559 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830534.533952:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb992315a40 x1874672572691968/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830559 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830534.533968:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830534.533972:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787830534.533976:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb992315a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572691968:192.168.203.129@tcp:38:kworker.0 00010000:00010000:2.0:1787830535.363244:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787830536.387232:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787830537.414365:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787830538.435978:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787830539.458931:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787830539.652309:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787830539.652319:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787830539.652324:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787830539.652333:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 3491 00000100:00080000:2.0:1787830539.652344:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787830539.652352:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:2.0:1787830539.652386:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8e940 x1874672572692096/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787830539.652417:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787830539.652421:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787830539.652435:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787830539.652440:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 3491 00000100:00080000:2.0:1787830539.652444:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787830539.652448:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:2.0:1787830539.652459:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787830539.652471:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8f840 x1874672572692224/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787830539.652481:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787830539.652484:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1787830539.652486:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787830539.652490:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 3491 00000100:00080000:2.0:1787830539.652493:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787830539.652497:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:2.0:1787830539.652502:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787830539.652513:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8ed00 x1874672572692352/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787830539.652597:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8e940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572692096:192.168.203.129@tcp:250:kworker.0 00000800:00000100:0.0:1787830539.652651:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787830539.652686:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830539/real 1787830539] req@ffff8cb998a8e940 x1874672572692096/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830564 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830539.652701:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8e940 x1874672572692096/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830564 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830539.652710:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830539.652714:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787830539.652718:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8e940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572692096:192.168.203.129@tcp:250:kworker.0 00000100:00100000:0.0:1787830539.652724:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8f840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572692224:192.168.203.129@tcp:38:kworker.0 00000800:00000100:0.0:1787830539.652744:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787830539.652754:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830539/real 1787830539] req@ffff8cb998a8f840 x1874672572692224/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830564 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830539.652765:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8f840 x1874672572692224/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830564 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830539.652772:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830539.652774:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787830539.652777:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8f840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572692224:192.168.203.129@tcp:38:kworker.0 00000100:00100000:0.0:1787830539.652781:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8ed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572692352:192.168.203.129@tcp:38:kworker.0 00000800:00000100:0.0:1787830539.652797:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787830539.652806:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830539/real 1787830539] req@ffff8cb998a8ed00 x1874672572692352/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830564 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830539.652829:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8ed00 x1874672572692352/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830564 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830539.652836:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830539.652839:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787830539.652842:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8ed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572692352:192.168.203.129@tcp:38:kworker.0 00010000:00010000:0.0:1787830540.482526:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787830541.506375:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787830542.530452:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787830543.554526:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787830544.578745:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787830544.770447:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787830544.770467:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787830544.770471:0:37:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787830544.770481:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 3496 00000100:00080000:3.0:1787830544.770492:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787830544.770500:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:3.0:1787830544.770536:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb9923170c0 x1874672572692480/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787830544.770566:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787830544.770570:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787830544.770572:0:37:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787830544.770576:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 3496 00000100:00080000:3.0:1787830544.770580:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787830544.770584:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:3.0:1787830544.770593:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787830544.770613:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb9923170c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572692480:192.168.203.129@tcp:250:kworker.0 00000100:00100000:3.0:1787830544.770620:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb992315a40 x1874672572692608/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787830544.770633:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787830544.770636:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1787830544.770638:0:37:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787830544.770642:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 3496 00000100:00080000:3.0:1787830544.770658:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787830544.770663:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:3.0:1787830544.770669:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787830544.770681:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb9923161c0 x1874672572692736/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1787830544.770692:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787830544.770735:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830544/real 1787830544] req@ffff8cb9923170c0 x1874672572692480/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830569 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830544.770751:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb9923170c0 x1874672572692480/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830569 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830544.770761:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830544.770765:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787830544.770768:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb9923170c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572692480:192.168.203.129@tcp:250:kworker.0 00000100:00100000:0.0:1787830544.770790:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb992315a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572692608:192.168.203.129@tcp:38:kworker.0 00000800:00000100:0.0:1787830544.770810:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787830544.770822:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830544/real 1787830544] req@ffff8cb992315a40 x1874672572692608/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830569 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830544.770833:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb992315a40 x1874672572692608/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830569 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830544.770848:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830544.770851:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787830544.770855:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb992315a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572692608:192.168.203.129@tcp:38:kworker.0 00000100:00100000:0.0:1787830544.770875:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb9923161c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572692736:192.168.203.129@tcp:38:kworker.0 00000800:00000100:0.0:1787830544.770901:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787830544.770917:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830544/real 1787830544] req@ffff8cb9923161c0 x1874672572692736/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830569 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830544.770928:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb9923161c0 x1874672572692736/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830569 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830544.770935:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830544.770938:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787830544.770941:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb9923161c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572692736:192.168.203.129@tcp:38:kworker.0 00010000:00010000:1.0:1787830545.602879:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787830546.627361:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787830547.650498:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787830548.674592:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787830549.698850:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787830549.891122:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787830549.891133:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787830549.891136:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787830549.891146:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 3502 00000100:00080000:1.0:1787830549.891157:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787830549.891176:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:1.0:1787830549.891211:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f3b480 x1874672572692864/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787830549.891239:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787830549.891258:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787830549.891284:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787830549.891288:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 3502 00000100:00080000:1.0:1787830549.891292:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787830549.891296:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:1.0:1787830549.891306:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787830549.891319:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f3b840 x1874672572692992/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787830549.891328:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787830549.891331:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1787830549.891332:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787830549.891336:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 3502 00000100:00080000:1.0:1787830549.891338:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787830549.891341:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:1.0:1787830549.891346:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787830549.891356:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f39a40 x1874672572693120/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787830549.891543:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f3b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572692864:192.168.203.129@tcp:250:kworker.0 00000800:00000100:0.0:1787830549.891607:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787830549.891654:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830549/real 1787830549] req@ffff8cb999f3b480 x1874672572692864/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830574 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830549.891670:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f3b480 x1874672572692864/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830574 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830549.891680:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830549.891689:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787830549.891693:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f3b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572692864:192.168.203.129@tcp:250:kworker.0 00000100:00100000:0.0:1787830549.891700:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f3b840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572692992:192.168.203.129@tcp:38:kworker.0 00000800:00000100:0.0:1787830549.891723:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787830549.891735:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830549/real 1787830549] req@ffff8cb999f3b840 x1874672572692992/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830574 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830549.891746:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f3b840 x1874672572692992/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830574 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830549.891753:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830549.891756:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787830549.891758:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f3b840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572692992:192.168.203.129@tcp:38:kworker.0 00000100:00100000:0.0:1787830549.891764:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f39a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572693120:192.168.203.129@tcp:38:kworker.0 00000800:00000100:0.0:1787830549.891783:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787830549.891794:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830549/real 1787830549] req@ffff8cb999f39a40 x1874672572693120/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830574 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830549.891805:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f39a40 x1874672572693120/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830574 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830549.891813:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830549.891816:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787830549.891819:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f39a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572693120:192.168.203.129@tcp:38:kworker.0 00010000:00010000:1.0:1787830550.722444:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787830551.746906:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787830552.771910:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787830553.794832:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787830554.818624:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787830555.010627:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787830555.010637:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787830555.010641:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787830555.010649:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 3507 00000100:00080000:2.0:1787830555.010659:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787830555.010675:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:2.0:1787830555.010706:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8cb40 x1874672572693248/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787830555.010755:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787830555.010758:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787830555.010760:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787830555.010764:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 3507 00000100:00080000:2.0:1787830555.010767:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787830555.010771:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:2.0:1787830555.010780:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787830555.010802:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8c000 x1874672572693376/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787830555.010811:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787830555.010814:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1787830555.010815:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787830555.010819:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 3507 00000100:00080000:2.0:1787830555.010822:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787830555.010826:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:2.0:1787830555.010831:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787830555.010849:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8cb40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572693248:192.168.203.129@tcp:250:kworker.0 00000100:00100000:2.0:1787830555.010855:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8e580 x1874672572693504/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1787830555.010913:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787830555.010963:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830555/real 1787830555] req@ffff8cb998a8cb40 x1874672572693248/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830580 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830555.010990:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8cb40 x1874672572693248/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830580 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830555.010999:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830555.011003:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787830555.011006:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8cb40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572693248:192.168.203.129@tcp:250:kworker.0 00000100:00100000:0.0:1787830555.011013:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572693376:192.168.203.129@tcp:38:kworker.0 00000800:00000100:0.0:1787830555.011035:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787830555.011047:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830555/real 1787830555] req@ffff8cb998a8c000 x1874672572693376/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830580 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830555.011059:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8c000 x1874672572693376/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830580 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830555.011067:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830555.011070:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787830555.011073:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572693376:192.168.203.129@tcp:38:kworker.0 00000100:00100000:0.0:1787830555.011096:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8e580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572693504:192.168.203.129@tcp:38:kworker.0 00000800:00000100:0.0:1787830555.011115:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787830555.011125:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830555/real 1787830555] req@ffff8cb998a8e580 x1874672572693504/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830580 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830555.011153:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8e580 x1874672572693504/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830580 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830555.011162:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830555.011165:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787830555.011168:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8e580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572693504:192.168.203.129@tcp:38:kworker.0 00010000:00010000:0.0:1787830555.842448:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787830556.866611:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787830557.890791:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787830558.914633:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787830559.938550:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787830560.130632:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787830560.130654:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787830560.130657:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787830560.130664:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 3512 00000100:00080000:1.0:1787830560.130672:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787830560.130677:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:1.0:1787830560.130706:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f39e00 x1874672572693632/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787830560.130736:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787830560.130740:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787830560.130741:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787830560.130745:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 3512 00000100:00080000:1.0:1787830560.130748:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787830560.130752:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:1.0:1787830560.130769:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787830560.130782:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f39a40 x1874672572693760/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787830560.130791:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787830560.130794:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1787830560.130795:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787830560.130798:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 3512 00000100:00080000:1.0:1787830560.130802:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787830560.130805:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:1.0:1787830560.130808:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787830560.130818:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f3b840 x1874672572693888/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787830560.131325:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f39e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572693632:192.168.203.129@tcp:250:kworker.0 00000800:00000100:0.0:1787830560.131385:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787830560.131433:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830560/real 1787830560] req@ffff8cb999f39e00 x1874672572693632/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830585 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830560.131450:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f39e00 x1874672572693632/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830585 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830560.131459:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830560.131463:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787830560.131467:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f39e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572693632:192.168.203.129@tcp:250:kworker.0 00000100:00100000:0.0:1787830560.131474:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f39a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572693760:192.168.203.129@tcp:38:kworker.0 00000800:00000100:0.0:1787830560.131496:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787830560.131509:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830560/real 1787830560] req@ffff8cb999f39a40 x1874672572693760/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830585 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830560.131617:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f39a40 x1874672572693760/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830585 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830560.131633:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830560.131636:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787830560.131640:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f39a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572693760:192.168.203.129@tcp:38:kworker.0 00000100:00100000:0.0:1787830560.131646:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f3b840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572693888:192.168.203.129@tcp:38:kworker.0 00000800:00000100:0.0:1787830560.131668:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787830560.131680:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830560/real 1787830560] req@ffff8cb999f3b840 x1874672572693888/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830585 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830560.131691:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f3b840 x1874672572693888/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830585 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830560.131699:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830560.131702:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787830560.131705:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f3b840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572693888:192.168.203.129@tcp:38:kworker.0 00010000:00010000:1.0:1787830560.962653:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787830561.986372:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787830563.010498:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787830564.034580:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787830565.058904:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787830565.250802:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787830565.250813:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787830565.250817:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787830565.250827:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 3517 00000100:00080000:2.0:1787830565.251053:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787830565.251061:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:2.0:1787830565.251095:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8d680 x1874672572694016/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787830565.251126:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787830565.251130:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787830565.251132:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787830565.251136:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 3517 00000100:00080000:2.0:1787830565.251139:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787830565.251143:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:2.0:1787830565.251153:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787830565.251165:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8f480 x1874672572694144/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787830565.251173:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8d680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572694016:192.168.203.129@tcp:250:kworker.0 00000100:00080000:2.0:1787830565.251179:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787830565.251182:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1787830565.251184:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787830565.251192:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 3517 00000100:00080000:2.0:1787830565.251196:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787830565.251200:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:2.0:1787830565.251205:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787830565.251216:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8cf00 x1874672572694272/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1787830565.251285:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787830565.251325:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830565/real 1787830565] req@ffff8cb998a8d680 x1874672572694016/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830590 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830565.251341:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8d680 x1874672572694016/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830590 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830565.251351:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830565.251365:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787830565.251369:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8d680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572694016:192.168.203.129@tcp:250:kworker.0 00000100:00100000:0.0:1787830565.251394:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572694144:192.168.203.129@tcp:38:kworker.0 00000800:00000100:0.0:1787830565.251419:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787830565.251433:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830565/real 1787830565] req@ffff8cb998a8f480 x1874672572694144/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830590 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830565.251453:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8f480 x1874672572694144/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830590 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830565.251462:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830565.251465:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787830565.251468:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572694144:192.168.203.129@tcp:38:kworker.0 00000100:00100000:0.0:1787830565.251475:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8cf00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572694272:192.168.203.129@tcp:38:kworker.0 00000800:00000100:0.0:1787830565.251496:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787830565.251509:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830565/real 1787830565] req@ffff8cb998a8cf00 x1874672572694272/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830590 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830565.251521:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8cf00 x1874672572694272/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830590 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830565.251529:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830565.251533:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787830565.251536:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8cf00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572694272:192.168.203.129@tcp:38:kworker.0 00010000:00010000:1.0:1787830566.084495:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787830567.106457:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787830568.130440:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787830569.156902:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787830570.178663:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787830570.370525:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787830570.370549:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787830570.370552:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787830570.370562:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 3522 00000100:00080000:2.0:1787830570.370572:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787830570.370579:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:2.0:1787830570.370613:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8fc00 x1874672572694400/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787830570.370641:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787830570.370645:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787830570.370648:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787830570.370653:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 3522 00000100:00080000:2.0:1787830570.370657:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787830570.370662:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:2.0:1787830570.370672:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787830570.370697:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8c780 x1874672572694528/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1787830570.370699:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8fc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572694400:192.168.203.129@tcp:250:kworker.0 00000100:00080000:2.0:1787830570.370716:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787830570.370720:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1787830570.370722:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787830570.370726:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 3522 00000100:00080000:2.0:1787830570.370730:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787830570.370735:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:2.0:1787830570.370740:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787830570.370752:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8f0c0 x1874672572694656/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1787830570.370770:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:3.0:1787830570.370812:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830570/real 1787830570] req@ffff8cb998a8fc00 x1874672572694400/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830595 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787830570.370830:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8fc00 x1874672572694400/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830595 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787830570.370841:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787830570.370845:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:3.0:1787830570.370849:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8fc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572694400:192.168.203.129@tcp:250:kworker.0 00000100:00100000:3.0:1787830570.370873:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8c780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572694528:192.168.203.129@tcp:38:kworker.0 00000800:00000100:3.0:1787830570.370899:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:3.0:1787830570.370912:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830570/real 1787830570] req@ffff8cb998a8c780 x1874672572694528/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830595 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787830570.370925:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8c780 x1874672572694528/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830595 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787830570.370933:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787830570.370936:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:3.0:1787830570.370947:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8c780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572694528:192.168.203.129@tcp:38:kworker.0 00000100:00100000:3.0:1787830570.370953:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8f0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572694656:192.168.203.129@tcp:38:kworker.0 00000800:00000100:3.0:1787830570.370976:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:3.0:1787830570.370988:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830570/real 1787830570] req@ffff8cb998a8f0c0 x1874672572694656/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830595 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787830570.370999:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8f0c0 x1874672572694656/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830595 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787830570.371007:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787830570.371010:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:3.0:1787830570.371013:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8f0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572694656:192.168.203.129@tcp:38:kworker.0 00010000:00010000:0.0:1787830571.204290:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787830572.226901:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787830573.250523:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787830574.274550:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787830575.298616:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787830575.491303:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787830575.491314:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787830575.491317:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787830575.491327:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 3527 00000100:00080000:1.0:1787830575.491336:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787830575.491341:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:1.0:1787830575.491376:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f3b480 x1874672572694784/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787830575.491406:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787830575.491409:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787830575.491411:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787830575.491415:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 3527 00000100:00080000:1.0:1787830575.491419:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787830575.491423:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:1.0:1787830575.491432:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787830575.491458:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f3b0c0 x1874672572694912/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787830575.491470:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787830575.491473:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1787830575.491476:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787830575.491480:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 3527 00000100:00080000:1.0:1787830575.491483:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787830575.491487:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:1.0:1787830575.491492:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787830575.491502:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999f38b40 x1874672572695040/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787830575.491581:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f3b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572694784:192.168.203.129@tcp:250:kworker.0 00000100:00100000:0.0:1787830575.491725:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f3b0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572694912:192.168.203.129@tcp:38:kworker.0 00000100:00100000:0.0:1787830575.491751:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999f38b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572695040:192.168.203.129@tcp:38:kworker.0 00010000:00010000:1.0:1787830576.323387:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787830577.346685:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787830578.370650:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:3.0:1787830578.564952:0:2381:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.129:988 00000400:00000100:3.0:1787830578.565001:0:2381:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.129@tcp at host 192.168.203.129 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:3.0:1787830578.565018:0:2381:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.129@tcp 00000800:00000100:3.0:1787830578.565062:0:2381:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.129@tcp 00000800:00000100:3.0:1787830578.565072:0:2381:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.129@tcp 00000100:00100000:0.0:1787830578.565439:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830575/real 1787830578] req@ffff8cb999f3b480 x1874672572694784/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830600 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830578.565464:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f3b480 x1874672572694784/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830600 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830578.565474:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830578.565479:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787830578.565483:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f3b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572694784:192.168.203.129@tcp:250:kworker.0 00000100:00100000:0.0:1787830578.565490:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830575/real 1787830578] req@ffff8cb999f3b0c0 x1874672572694912/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830600 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830578.565501:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f3b0c0 x1874672572694912/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830600 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830578.565508:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830578.565511:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787830578.565513:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f3b0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572694912:192.168.203.129@tcp:38:kworker.0 00000100:00100000:0.0:1787830578.565520:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830575/real 1787830578] req@ffff8cb999f38b40 x1874672572695040/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830600 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830578.565556:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999f38b40 x1874672572695040/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830600 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830578.565565:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830578.565569:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787830578.565572:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999f38b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572695040:192.168.203.129@tcp:38:kworker.0 00010000:00010000:1.0:1787830579.396679:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787830580.419666:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787830580.610417:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787830580.610427:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787830580.610431:0:37:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787830580.610440:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 3532 00000100:00080000:2.0:1787830580.610451:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787830580.610458:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:2.0:1787830580.610505:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8e1c0 x1874672572695168/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787830580.610536:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787830580.610540:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787830580.610542:0:37:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787830580.610546:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 3532 00000100:00080000:2.0:1787830580.610550:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787830580.610553:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:2.0:1787830580.610563:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787830580.610575:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8da40 x1874672572695296/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787830580.610586:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787830580.610589:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1787830580.610591:0:37:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1787830580.610599:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8e1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572695168:192.168.203.129@tcp:250:kworker.0 00000100:00080000:2.0:1787830580.610619:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 3532 00000100:00080000:2.0:1787830580.610624:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787830580.610628:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:2.0:1787830580.610647:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787830580.610660:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8de00 x1874672572695424/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1787830580.610672:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787830580.610710:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830580/real 1787830580] req@ffff8cb998a8e1c0 x1874672572695168/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830605 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830580.610725:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8e1c0 x1874672572695168/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830605 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830580.610735:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830580.610739:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787830580.610742:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8e1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572695168:192.168.203.129@tcp:250:kworker.0 00000100:00100000:0.0:1787830580.610749:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8da40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572695296:192.168.203.129@tcp:38:kworker.0 00000800:00000100:0.0:1787830580.610771:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787830580.610783:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830580/real 1787830580] req@ffff8cb998a8da40 x1874672572695296/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830605 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830580.610795:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8da40 x1874672572695296/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830605 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830580.610814:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830580.610818:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787830580.610822:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8da40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572695296:192.168.203.129@tcp:38:kworker.0 00000100:00100000:0.0:1787830580.610847:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8de00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572695424:192.168.203.129@tcp:38:kworker.0 00000800:00000100:0.0:1787830580.610869:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787830580.610880:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830580/real 1787830580] req@ffff8cb998a8de00 x1874672572695424/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830605 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830580.610891:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8de00 x1874672572695424/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830605 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830580.610898:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830580.610904:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787830580.610906:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8de00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572695424:192.168.203.129@tcp:38:kworker.0 00010000:00010000:1.0:1787830581.442527:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787830582.467846:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787830583.490472:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787830584.514794:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787830585.539144:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787830585.730443:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787830585.730453:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787830585.730458:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787830585.730468:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 3537 00000100:00080000:3.0:1787830585.730478:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787830585.730485:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:3.0:1787830585.730520:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb992316940 x1874672572695552/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787830585.730566:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787830585.730570:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787830585.730571:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787830585.730584:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 3537 00000100:00080000:3.0:1787830585.730589:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787830585.730593:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:3.0:1787830585.730607:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787830585.730620:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb992316940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572695552:192.168.203.129@tcp:250:kworker.0 00000100:00100000:3.0:1787830585.730622:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb992315e00 x1874672572695680/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787830585.730638:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787830585.730641:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1787830585.730643:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787830585.730647:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 3537 00000100:00080000:3.0:1787830585.730651:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787830585.730655:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:3.0:1787830585.730660:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787830585.730675:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb992314f00 x1874672572695808/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1787830585.730691:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787830585.730730:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830585/real 1787830585] req@ffff8cb992316940 x1874672572695552/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830610 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830585.730744:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb992316940 x1874672572695552/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830610 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830585.730754:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830585.730758:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787830585.730768:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb992316940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572695552:192.168.203.129@tcp:250:kworker.0 00000100:00100000:0.0:1787830585.730792:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb992315e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572695680:192.168.203.129@tcp:38:kworker.0 00000800:00000100:0.0:1787830585.730816:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787830585.730828:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830585/real 1787830585] req@ffff8cb992315e00 x1874672572695680/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830610 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830585.730839:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb992315e00 x1874672572695680/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830610 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830585.731167:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830585.731172:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787830585.731176:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb992315e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572695680:192.168.203.129@tcp:38:kworker.0 00000100:00100000:0.0:1787830585.731184:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb992314f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572695808:192.168.203.129@tcp:38:kworker.0 00000800:00000100:0.0:1787830585.731222:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787830585.731235:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830585/real 1787830585] req@ffff8cb992314f00 x1874672572695808/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830610 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830585.731247:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb992314f00 x1874672572695808/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830610 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830585.731254:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830585.731258:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787830585.731292:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb992314f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572695808:192.168.203.129@tcp:38:kworker.0 00010000:00010000:0.0:1787830586.562694:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787830587.587079:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787830588.610550:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787830589.634433:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787830590.658531:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787830590.850489:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787830590.850498:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787830590.850502:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787830590.850511:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 3543 00000100:00080000:3.0:1787830590.850792:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787830590.850799:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:3.0:1787830590.850830:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb992314780 x1874672572695936/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787830590.850859:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787830590.850863:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787830590.850864:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787830590.850868:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 3543 00000100:00080000:3.0:1787830590.850871:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787830590.850874:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:3.0:1787830590.850883:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787830590.850893:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb992314b40 x1874672572696064/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787830590.850902:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787830590.850922:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1787830590.850924:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787830590.850928:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 3543 00000100:00080000:3.0:1787830590.850931:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787830590.850934:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:3.0:1787830590.850938:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787830590.850948:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb992317480 x1874672572696192/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787830590.852322:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb992314780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572695936:192.168.203.129@tcp:250:kworker.0 00000800:00000100:0.0:1787830590.852386:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787830590.852427:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830590/real 1787830590] req@ffff8cb992314780 x1874672572695936/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830615 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830590.852444:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb992314780 x1874672572695936/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830615 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830590.852455:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830590.852459:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787830590.852464:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb992314780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572695936:192.168.203.129@tcp:250:kworker.0 00000100:00100000:0.0:1787830590.852471:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb992314b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572696064:192.168.203.129@tcp:38:kworker.0 00000800:00000100:0.0:1787830590.852495:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787830590.852506:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830590/real 1787830590] req@ffff8cb992314b40 x1874672572696064/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830615 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830590.852519:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb992314b40 x1874672572696064/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830615 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830590.852526:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830590.852530:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787830590.852533:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb992314b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572696064:192.168.203.129@tcp:38:kworker.0 00000100:00100000:0.0:1787830590.852552:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb992317480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572696192:192.168.203.129@tcp:38:kworker.0 00000800:00000100:0.0:1787830590.852571:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787830590.852581:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830590/real 1787830590] req@ffff8cb992317480 x1874672572696192/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830615 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830590.852608:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb992317480 x1874672572696192/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830615 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830590.852617:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830590.852620:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787830590.852623:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb992317480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572696192:192.168.203.129@tcp:38:kworker.0 00010000:00010000:1.0:1787830591.683692:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787830592.709857:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787830593.730589:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787830594.754409:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787830595.778556:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787830595.970376:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787830595.970406:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787830595.970409:0:37:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787830595.970418:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 3548 00000100:00080000:3.0:1787830595.970429:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787830595.970436:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:3.0:1787830595.970466:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb992316580 x1874672572696320/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787830595.970557:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb992316580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572696320:192.168.203.129@tcp:250:kworker.0 00000800:00000100:0.0:1787830595.970615:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00080000:3.0:1787830595.970645:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787830595.970650:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787830595.970652:0:37:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787830595.970657:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 3548 00000100:00080000:3.0:1787830595.970662:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787830595.970667:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:3.0:1787830595.970677:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787830595.970698:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb992317480 x1874672572696448/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787830595.970753:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830595/real 1787830595] req@ffff8cb992316580 x1874672572696320/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830620 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830595.970769:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb992316580 x1874672572696320/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830620 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830595.970778:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830595.970782:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787830595.970785:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb992316580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572696320:192.168.203.129@tcp:250:kworker.0 00000100:00100000:0.0:1787830595.970807:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb992317480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572696448:192.168.203.129@tcp:38:kworker.0 00000800:00000100:0.0:1787830595.970829:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787830595.970840:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830595/real 1787830595] req@ffff8cb992317480 x1874672572696448/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830620 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830595.970851:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb992317480 x1874672572696448/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830620 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830595.970865:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830595.970869:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787830595.970872:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb992317480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572696448:192.168.203.129@tcp:38:kworker.0 00000100:00080000:3.0:1787830595.970960:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787830595.970965:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1787830595.970967:0:37:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787830595.970971:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 3548 00000100:00080000:3.0:1787830595.970975:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787830595.970978:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:3.0:1787830595.970984:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787830595.970996:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb992314b40 x1874672572696576/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787830595.971044:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb992314b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572696576:192.168.203.129@tcp:38:kworker.0 00000800:00000100:0.0:1787830595.971066:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787830595.971131:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830595/real 1787830595] req@ffff8cb992314b40 x1874672572696576/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830620 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830595.971143:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb992314b40 x1874672572696576/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830620 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830595.971151:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830595.971155:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787830595.971158:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb992314b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572696576:192.168.203.129@tcp:38:kworker.0 00010000:00010000:1.0:1787830596.802912:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787830597.827756:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787830598.850677:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787830599.875661:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787830600.898462:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787830601.090365:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787830601.090376:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787830601.090380:0:37:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787830601.090389:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 3553 00000100:00080000:3.0:1787830601.090400:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787830601.090407:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:3.0:1787830601.090442:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb992314780 x1874672572696704/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787830601.090525:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb992314780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572696704:192.168.203.129@tcp:250:kworker.0 00000800:00000100:0.0:1787830601.090584:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00080000:3.0:1787830601.090617:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787830601.090621:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787830601.090623:0:37:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787830601.090628:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 3553 00000100:00080000:3.0:1787830601.090632:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787830601.090636:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:3.0:1787830601.090645:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787830601.090658:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb992314f00 x1874672572696832/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787830601.090713:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830601/real 1787830601] req@ffff8cb992314780 x1874672572696704/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830626 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830601.090730:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb992314780 x1874672572696704/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830626 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830601.090739:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830601.090743:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787830601.090760:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb992314780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572696704:192.168.203.129@tcp:250:kworker.0 00000100:00100000:0.0:1787830601.090784:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb992314f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572696832:192.168.203.129@tcp:38:kworker.0 00000800:00000100:0.0:1787830601.090808:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787830601.090821:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830601/real 1787830601] req@ffff8cb992314f00 x1874672572696832/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830626 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830601.090832:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb992314f00 x1874672572696832/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830626 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830601.090840:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830601.090843:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787830601.090846:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb992314f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572696832:192.168.203.129@tcp:38:kworker.0 00000100:00080000:3.0:1787830601.090934:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787830601.090938:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1787830601.090940:0:37:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787830601.090945:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 3553 00000100:00080000:3.0:1787830601.090949:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787830601.090953:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:3.0:1787830601.090964:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787830601.090976:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb992315e00 x1874672572696960/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787830601.091021:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb992315e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572696960:192.168.203.129@tcp:38:kworker.0 00000800:00000100:0.0:1787830601.091058:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787830601.091120:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830601/real 1787830601] req@ffff8cb992315e00 x1874672572696960/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830626 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830601.091131:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb992315e00 x1874672572696960/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830626 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830601.091139:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830601.091142:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787830601.091145:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb992315e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572696960:192.168.203.129@tcp:38:kworker.0 00010000:00010000:0.0:1787830601.922649:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787830602.946982:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787830603.970491:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787830604.994482:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787830606.019297:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787830606.210455:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787830606.210465:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787830606.210469:0:37:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787830606.210479:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 3558 00000100:00080000:2.0:1787830606.210491:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787830606.210499:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:2.0:1787830606.210533:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8d2c0 x1874672572697088/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787830606.210565:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787830606.210568:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787830606.210570:0:37:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787830606.210574:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 3558 00000100:00080000:2.0:1787830606.210578:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787830606.210581:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:2.0:1787830606.210591:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787830606.210604:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8c3c0 x1874672572697216/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787830606.210614:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787830606.210617:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1787830606.210619:0:37:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787830606.210622:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 3558 00000100:00080000:2.0:1787830606.210639:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787830606.210644:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:2.0:1787830606.210649:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787830606.210662:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8de00 x1874672572697344/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787830606.211206:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8d2c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572697088:192.168.203.129@tcp:250:kworker.0 00000800:00000100:0.0:1787830606.211289:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787830606.211324:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830606/real 1787830606] req@ffff8cb998a8d2c0 x1874672572697088/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830631 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830606.211339:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8d2c0 x1874672572697088/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830631 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830606.211348:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830606.211352:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787830606.211356:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8d2c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572697088:192.168.203.129@tcp:250:kworker.0 00000100:00100000:0.0:1787830606.211363:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8c3c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572697216:192.168.203.129@tcp:38:kworker.0 00000800:00000100:0.0:1787830606.211386:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787830606.211398:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830606/real 1787830606] req@ffff8cb998a8c3c0 x1874672572697216/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830631 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830606.211410:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8c3c0 x1874672572697216/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830631 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830606.211417:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830606.211420:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787830606.211424:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8c3c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572697216:192.168.203.129@tcp:38:kworker.0 00000100:00100000:0.0:1787830606.211429:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8de00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572697344:192.168.203.129@tcp:38:kworker.0 00000800:00000100:0.0:1787830606.211926:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787830606.211941:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830606/real 1787830606] req@ffff8cb998a8de00 x1874672572697344/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830631 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830606.212039:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8de00 x1874672572697344/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830631 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830606.212049:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830606.212053:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787830606.212056:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8de00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572697344:192.168.203.129@tcp:38:kworker.0 00010000:00010000:1.0:1787830607.042523:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787830608.066468:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787830609.090574:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787830610.115950:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787830611.138452:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787830611.331245:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787830611.331255:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787830611.331287:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787830611.331298:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 3563 00000100:00080000:0.0:1787830611.331309:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787830611.331625:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:0.0:1787830611.331669:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c6b480 x1874672572697472/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1787830611.331748:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c6b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572697472:192.168.203.129@tcp:250:kworker.0 00000800:00000100:3.0:1787830611.331806:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00080000:0.0:1787830611.331838:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787830611.331852:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787830611.331854:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787830611.331859:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 3563 00000100:00080000:0.0:1787830611.331864:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787830611.331868:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:0.0:1787830611.331877:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787830611.331903:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c6bc00 x1874672572697600/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1787830611.332127:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830611/real 1787830611] req@ffff8cb999c6b480 x1874672572697472/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830636 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787830611.332145:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c6b480 x1874672572697472/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830636 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787830611.332154:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787830611.332158:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:3.0:1787830611.332163:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c6b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572697472:192.168.203.129@tcp:250:kworker.0 00000100:00100000:3.0:1787830611.332185:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c6bc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572697600:192.168.203.129@tcp:38:kworker.0 00000800:00000100:3.0:1787830611.332207:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:3.0:1787830611.332219:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830611/real 1787830611] req@ffff8cb999c6bc00 x1874672572697600/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830636 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787830611.332231:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c6bc00 x1874672572697600/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830636 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787830611.332239:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787830611.332242:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:3.0:1787830611.332245:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c6bc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572697600:192.168.203.129@tcp:38:kworker.0 00000100:00080000:0.0:1787830611.334572:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787830611.334582:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1787830611.334586:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787830611.334594:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 3563 00000100:00080000:0.0:1787830611.334603:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787830611.334609:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:0.0:1787830611.334621:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787830611.334650:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c69a40 x1874672572697728/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1787830611.334699:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c69a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572697728:192.168.203.129@tcp:38:kworker.0 00000800:00000100:3.0:1787830611.334762:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:3.0:1787830611.334796:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830611/real 1787830611] req@ffff8cb999c69a40 x1874672572697728/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830636 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787830611.334816:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c69a40 x1874672572697728/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830636 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787830611.334827:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787830611.334830:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:3.0:1787830611.334834:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c69a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572697728:192.168.203.129@tcp:38:kworker.0 00010000:00010000:1.0:1787830612.162596:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787830613.188567:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787830614.211036:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787830615.234411:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787830616.259325:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787830616.450392:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787830616.450401:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787830616.450404:0:37:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787830616.450414:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 3568 00000100:00080000:3.0:1787830616.450425:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787830616.450433:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:3.0:1787830616.450468:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb992316940 x1874672572697856/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787830616.450496:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787830616.450498:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787830616.450500:0:37:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787830616.450502:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 3568 00000100:00080000:3.0:1787830616.450505:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787830616.450507:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:3.0:1787830616.450515:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787830616.450539:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb9923161c0 x1874672572697984/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787830616.450549:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787830616.450552:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1787830616.450554:0:37:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787830616.450558:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 3568 00000100:00080000:3.0:1787830616.450562:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787830616.450565:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:3.0:1787830616.450570:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787830616.450580:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb992315a40 x1874672572698112/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787830616.450655:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb992316940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572697856:192.168.203.129@tcp:250:kworker.0 00000800:00000100:0.0:1787830616.450715:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787830616.450749:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830616/real 1787830616] req@ffff8cb992316940 x1874672572697856/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830641 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830616.450765:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb992316940 x1874672572697856/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830641 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830616.450775:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830616.450778:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787830616.450782:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb992316940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572697856:192.168.203.129@tcp:250:kworker.0 00000100:00100000:0.0:1787830616.450789:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb9923161c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572697984:192.168.203.129@tcp:38:kworker.0 00000800:00000100:0.0:1787830616.450819:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787830616.450831:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830616/real 1787830616] req@ffff8cb9923161c0 x1874672572697984/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830641 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830616.450842:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb9923161c0 x1874672572697984/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830641 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830616.450849:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830616.450852:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787830616.450855:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb9923161c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572697984:192.168.203.129@tcp:38:kworker.0 00000100:00100000:0.0:1787830616.450861:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb992315a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572698112:192.168.203.129@tcp:38:kworker.0 00000800:00000100:0.0:1787830616.450880:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1787830616.450891:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830616/real 1787830616] req@ffff8cb992315a40 x1874672572698112/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830641 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787830616.450901:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb992315a40 x1874672572698112/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830641 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787830616.450909:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787830616.450912:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:0.0:1787830616.450915:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb992315a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572698112:192.168.203.129@tcp:38:kworker.0 00010000:00010000:0.0:1787830617.282528:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787830618.306848:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787830619.330482:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787830620.355407:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787830621.378630:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787830621.570727:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787830621.570737:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787830621.570741:0:37:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787830621.570765:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 3573 00000100:00080000:0.0:1787830621.570776:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787830621.570784:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:0.0:1787830621.570815:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c68f00 x1874672572698240/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1787830621.570891:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c68f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572698240:192.168.203.129@tcp:250:kworker.0 00000800:00000100:3.0:1787830621.570952:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00080000:0.0:1787830621.570985:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787830621.570989:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787830621.570991:0:37:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787830621.570995:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 3573 00000100:00080000:0.0:1787830621.571000:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787830621.571004:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:0.0:1787830621.571012:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787830621.571024:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c6b840 x1874672572698368/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1787830621.571379:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830621/real 1787830621] req@ffff8cb999c68f00 x1874672572698240/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830646 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787830621.571398:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c68f00 x1874672572698240/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830646 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787830621.571408:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787830621.571412:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:3.0:1787830621.571416:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c68f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572698240:192.168.203.129@tcp:250:kworker.0 00000100:00100000:3.0:1787830621.571440:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c6b840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572698368:192.168.203.129@tcp:38:kworker.0 00000800:00000100:3.0:1787830621.571467:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:3.0:1787830621.571489:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830621/real 1787830621] req@ffff8cb999c6b840 x1874672572698368/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830646 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787830621.571501:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c6b840 x1874672572698368/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830646 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787830621.571509:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787830621.571513:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:3.0:1787830621.571516:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c6b840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572698368:192.168.203.129@tcp:38:kworker.0 00000100:00080000:0.0:1787830621.572039:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787830621.572045:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1787830621.572047:0:37:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787830621.572052:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 3573 00000100:00080000:0.0:1787830621.572057:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787830621.572060:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:0.0:1787830621.572067:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787830621.572082:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c6b840 x1874672572698496/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1787830621.572140:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c6b840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572698496:192.168.203.129@tcp:38:kworker.0 00000800:00000100:3.0:1787830621.572173:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:3.0:1787830621.572190:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830621/real 1787830621] req@ffff8cb999c6b840 x1874672572698496/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830646 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787830621.572204:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c6b840 x1874672572698496/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830646 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787830621.572213:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787830621.572218:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:3.0:1787830621.572222:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c6b840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572698496:192.168.203.129@tcp:38:kworker.0 00010000:00010000:1.0:1787830622.402638:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787830623.427072:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787830624.450493:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787830625.475482:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787830626.499444:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787830626.690386:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787830626.690397:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787830626.690401:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787830626.690412:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 3578 00000100:00080000:0.0:1787830626.690424:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787830626.690431:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:0.0:1787830626.690536:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c68f00 x1874672572698624/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787830626.690571:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787830626.690575:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787830626.690577:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787830626.690581:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 3578 00000100:00080000:0.0:1787830626.690585:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787830626.690591:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:3.0:1787830626.690710:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c68f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572698624:192.168.203.129@tcp:250:kworker.0 00000800:00000100:3.0:1787830626.690785:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:3.0:1787830626.690829:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830626/real 1787830626] req@ffff8cb999c68f00 x1874672572698624/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830651 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787830626.690846:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c68f00 x1874672572698624/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830651 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787830626.690872:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787830626.690876:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:3.0:1787830626.690880:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c68f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572698624:192.168.203.129@tcp:250:kworker.0 00000008:00100000:0.0:1787830626.691024:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787830626.691042:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c69a40 x1874672572698752/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787830626.691114:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c69a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572698752:192.168.203.129@tcp:38:kworker.0 00000800:00000100:1.0:1787830626.691142:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00080000:0.0:1787830626.691183:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787830626.691187:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1787830626.691188:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787830626.691193:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 3578 00000100:00080000:0.0:1787830626.691203:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787830626.691208:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:0.0:1787830626.691213:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787830626.691224:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c6bc00 x1874672572698880/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787830626.691277:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830626/real 1787830626] req@ffff8cb999c69a40 x1874672572698752/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830651 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830626.691302:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c69a40 x1874672572698752/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830651 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830626.691310:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830626.691313:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787830626.691316:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c69a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572698752:192.168.203.129@tcp:38:kworker.0 00000100:00100000:1.0:1787830626.691329:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c6bc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572698880:192.168.203.129@tcp:38:kworker.0 00000800:00000100:1.0:1787830626.691358:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1787830626.691370:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830626/real 1787830626] req@ffff8cb999c6bc00 x1874672572698880/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830651 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830626.691380:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c6bc00 x1874672572698880/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830651 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830626.691387:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830626.691390:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:1.0:1787830626.691393:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c6bc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572698880:192.168.203.129@tcp:38:kworker.0 00010000:00010000:1.0:1787830627.523156:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787830628.546943:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787830629.570498:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787830630.594459:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787830631.618598:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787830631.810402:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787830631.810413:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787830631.810416:0:37:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787830631.810425:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 3583 00000100:00080000:0.0:1787830631.810435:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787830631.810442:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:0.0:1787830631.810476:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c6b480 x1874672572699008/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787830631.810507:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787830631.810510:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787830631.810512:0:37:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787830631.810515:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 3583 00000100:00080000:0.0:1787830631.810519:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787830631.810522:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:0.0:1787830631.810530:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787830631.810542:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c69e00 x1874672572699136/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787830631.810552:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787830631.810567:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1787830631.810569:0:37:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787830631.810574:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 3583 00000100:00080000:0.0:1787830631.810578:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787830631.810583:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:0.0:1787830631.810588:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787830631.810604:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb999c692c0 x1874672572699264/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1787830631.810721:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c6b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572699008:192.168.203.129@tcp:250:kworker.0 00000800:00000100:3.0:1787830631.810777:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:3.0:1787830631.810810:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830631/real 1787830631] req@ffff8cb999c6b480 x1874672572699008/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830656 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787830631.810822:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c6b480 x1874672572699008/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830656 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787830631.810828:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787830631.810831:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:3.0:1787830631.810833:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c6b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572699008:192.168.203.129@tcp:250:kworker.0 00000100:00100000:3.0:1787830631.810837:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c69e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572699136:192.168.203.129@tcp:38:kworker.0 00000800:00000100:3.0:1787830631.810854:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:3.0:1787830631.810865:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830631/real 1787830631] req@ffff8cb999c69e00 x1874672572699136/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830656 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787830631.810875:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c69e00 x1874672572699136/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830656 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787830631.810882:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787830631.810885:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:3.0:1787830631.810888:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c69e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572699136:192.168.203.129@tcp:38:kworker.0 00000100:00100000:3.0:1787830631.810902:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb999c692c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572699264:192.168.203.129@tcp:38:kworker.0 00000800:00000100:3.0:1787830631.810921:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:3.0:1787830631.810932:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830631/real 1787830631] req@ffff8cb999c692c0 x1874672572699264/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830656 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787830631.810942:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb999c692c0 x1874672572699264/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830656 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787830631.810949:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787830631.810952:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:3.0:1787830631.810955:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb999c692c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572699264:192.168.203.129@tcp:38:kworker.0 00010000:00010000:0.0:1787830632.642723:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787830633.667086:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787830634.691287:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787830635.714477:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787830636.738641:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787830636.931581:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0bb8b105-f59c-455a-b6b1-ce40efc3999c->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787830636.931591:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787830636.931595:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787830636.931605:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 3589 00000100:00080000:2.0:1787830636.931615:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787830636.931622:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.129@tcp: import ffff8cb98ab62800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:2.0:1787830636.931656:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8da40 x1874672572699392/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787830636.931687:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787830636.931691:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787830636.931693:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787830636.931697:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 3589 00000100:00080000:2.0:1787830636.931700:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787830636.931703:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8cb98543f000: import ffff8cb988ffa000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:2.0:1787830636.931712:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787830636.931724:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8e1c0 x1874672572699520/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787830636.931732:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8cef8de3-2326-42de-9d41-f4cb057ed6e8->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787830636.931749:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1787830636.931751:0:36:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787830636.931756:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: connect to NID 192.168.203.129@tcp last attempt 3589 00000100:00080000:2.0:1787830636.931759:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787830636.931763:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8cb98543f000: import ffff8cb998ab5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000008:00100000:2.0:1787830636.931768:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683201 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787830636.931781:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8cb998a8f0c0 x1874672572699648/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1787830636.931875:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8da40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572699392:192.168.203.129@tcp:250:kworker.0 00000800:00000100:3.0:1787830636.931934:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:3.0:1787830636.931978:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830636/real 1787830636] req@ffff8cb998a8da40 x1874672572699392/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830661 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787830636.931995:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8da40 x1874672572699392/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830661 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787830636.932007:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb98ab62800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787830636.932011:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:3.0:1787830636.932015:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8da40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0bb8b105-f59c-455a-b6b1-ce40efc3999c:2393:1874672572699392:192.168.203.129@tcp:250:kworker.0 00000100:00100000:3.0:1787830636.932023:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8e1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572699520:192.168.203.129@tcp:38:kworker.0 00000800:00000100:3.0:1787830636.932049:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:3.0:1787830636.932061:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830636/real 1787830636] req@ffff8cb998a8e1c0 x1874672572699520/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830661 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787830636.932082:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8e1c0 x1874672572699520/t0(0) o38->lustre-MDT0000-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830661 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787830636.932090:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb988ffa000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787830636.932094:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:3.0:1787830636.932097:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8e1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572699520:192.168.203.129@tcp:38:kworker.0 00000100:00100000:3.0:1787830636.932104:0:2393:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8cb998a8f0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572699648:192.168.203.129@tcp:38:kworker.0 00000800:00000100:3.0:1787830636.932126:0:2393:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:3.0:1787830636.932137:0:2393:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830636/real 1787830636] req@ffff8cb998a8f0c0 x1874672572699648/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830661 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787830636.932149:0:2393:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8cb998a8f0c0 x1874672572699648/t0(0) o38->lustre-MDT0001-mdc-ffff8cb98543f000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830661 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787830636.932157:0:2393:0:(import.c:65:import_set_state_nolock()) ffff8cb998ab5800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787830636.932160:0:2393:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:3.0:1787830636.932162:0:2393:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8cb998a8f0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8cef8de3-2326-42de-9d41-f4cb057ed6e8:2393:1874672572699648:192.168.203.129@tcp:38:kworker.0 00010000:00010000:1.0:1787830637.762515:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787830638.786608:0:6152:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787830639.811761:0:6153:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787830640.835338:0:6618:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) Debug log: 34138 lines, 34138 kept, 0 dropped, 0 bad.